From 75399837d57ff7267e8b89f3d6341b79780312d3 Mon Sep 17 00:00:00 2001 From: Nikita Golmgren Date: Tue, 21 Jan 2025 14:55:15 +0100 Subject: [PATCH 1/5] #14102 - experimental basic implementation of token filtering (no regex yet) --- .../DibiFluentPostgreDataSource.php | 30 +++++++++++++++-- src/Filter/FilterText.php | 32 +++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/DataSource/DibiFluentPostgreDataSource.php b/src/DataSource/DibiFluentPostgreDataSource.php index 19dd3d127..62bb5287b 100755 --- a/src/DataSource/DibiFluentPostgreDataSource.php +++ b/src/DataSource/DibiFluentPostgreDataSource.php @@ -26,7 +26,14 @@ public function applyFilterText(Filter\FilterText $filter) $or = []; foreach ($condition as $column => $value) { - + if ($filter->isSpecialChars()) { + if ($value === Filter\FilterText::TOKEN_EMPTY) { // Handle single '#' + $this->data_source->where("($column IS NULL OR $column = '')"); + continue; + } else if ($value === Filter\FilterText::TOKEN_EMPTY_ESCAPED) { // Handle '\#' for searching literal '#' + $value = Filter\FilterText::TOKEN_EMPTY; + } + } $column = '[' . $column . ']::varchar'; if ($filter->isExactSearch()) { @@ -42,7 +49,26 @@ public function applyFilterText(Filter\FilterText $filter) $x = []; foreach ($words as $word) { $escaped = $driver->escapeLike((string) $word, 0); - $x[] = "public.unaccent($column) ILIKE public.unaccent('%" . substr( $escaped, 1, -1) . "%')"; + if ($filter->isSpecialChars()) { + $allow_negation_filter = true; + if (strpos($word, Filter\FilterText::TOKEN_NEGATION_ESCAPED) !== false) { + //If the escaped negation token is in the beginning of text, explicitly forbid parsing it after it's replaced + if (strpos($word, Filter\FilterText::TOKEN_NEGATION_ESCAPED) !== false) { + $allow_negation_filter = false; + } + + $word = str_replace(Filter\FilterText::TOKEN_NEGATION_ESCAPED, Filter\FilterText::TOKEN_NEGATION, $word); + $escaped = $driver->escapeLike($word, 0); + } + + if ($allow_negation_filter && strpos($word, Filter\FilterText::TOKEN_NEGATION) !== false) { + //exclamation point means negation - the word is NOT included in the searched string + $x[] = "public.unaccent($column) NOT ILIKE public.unaccent('%" . substr($escaped, 2, -1) . "%')"; + continue; + } + } + + $x[] = "public.unaccent($column) ILIKE public.unaccent('%" . substr($escaped, 1, -1) . "%')"; } $or[] = "((" . implode(") AND (", $x) . "))"; } diff --git a/src/Filter/FilterText.php b/src/Filter/FilterText.php index ef305d362..fef61e4bc 100644 --- a/src/Filter/FilterText.php +++ b/src/Filter/FilterText.php @@ -12,6 +12,14 @@ class FilterText extends Filter { + /** Query that is exactly equal to '#' returns empty/null values */ + const TOKEN_EMPTY = '#'; + /** However, if your friend is named '#' and you really want to find him, you have to type this */ + const TOKEN_EMPTY_ESCAPED = '\#'; + /** Query that contains words that start with '!', excludes those words from search results */ + const TOKEN_NEGATION = '!'; + /** However, if your friend's name starts with '!', you have to type this */ + const TOKEN_NEGATION_ESCAPED = '\!'; /** * @var string @@ -33,6 +41,11 @@ class FilterText extends Filter */ protected $split_words_search = true; + /** + * @var bool + */ + protected $enable_special_chars = true; + /** * Adds text field to filter form @@ -102,4 +115,23 @@ public function hasSplitWordsSearch() { return $this->split_words_search; } + + /** + * @return bool + */ + public function isSpecialChars() + { + return $this->enable_special_chars; + } + + /** + * @param bool $enabled + * @return FilterText + */ + public function setSpecialChars($enabled) + { + $this->enable_special_chars = (bool) $enabled; + + return $this; + } } From f30d51332c07632c45347829679674d0f08ab337 Mon Sep 17 00:00:00 2001 From: Nikita Golmgren Date: Tue, 11 Feb 2025 13:43:25 +0100 Subject: [PATCH 2/5] #14102 Fix --- src/DataSource/DibiFluentPostgreDataSource.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/DataSource/DibiFluentPostgreDataSource.php b/src/DataSource/DibiFluentPostgreDataSource.php index 62bb5287b..daeb52fd9 100755 --- a/src/DataSource/DibiFluentPostgreDataSource.php +++ b/src/DataSource/DibiFluentPostgreDataSource.php @@ -30,9 +30,8 @@ public function applyFilterText(Filter\FilterText $filter) if ($value === Filter\FilterText::TOKEN_EMPTY) { // Handle single '#' $this->data_source->where("($column IS NULL OR $column = '')"); continue; - } else if ($value === Filter\FilterText::TOKEN_EMPTY_ESCAPED) { // Handle '\#' for searching literal '#' - $value = Filter\FilterText::TOKEN_EMPTY; } + $value = str_replace(Filter\FilterText::TOKEN_EMPTY_ESCAPED, Filter\FilterText::TOKEN_EMPTY, $value); } $column = '[' . $column . ']::varchar'; @@ -53,7 +52,7 @@ public function applyFilterText(Filter\FilterText $filter) $allow_negation_filter = true; if (strpos($word, Filter\FilterText::TOKEN_NEGATION_ESCAPED) !== false) { //If the escaped negation token is in the beginning of text, explicitly forbid parsing it after it's replaced - if (strpos($word, Filter\FilterText::TOKEN_NEGATION_ESCAPED) !== false) { + if (strpos($word, Filter\FilterText::TOKEN_NEGATION_ESCAPED) === 0) { $allow_negation_filter = false; } @@ -61,9 +60,10 @@ public function applyFilterText(Filter\FilterText $filter) $escaped = $driver->escapeLike($word, 0); } - if ($allow_negation_filter && strpos($word, Filter\FilterText::TOKEN_NEGATION) !== false) { + if ($allow_negation_filter && strpos($word, Filter\FilterText::TOKEN_NEGATION) === 0) { //exclamation point means negation - the word is NOT included in the searched string - $x[] = "public.unaccent($column) NOT ILIKE public.unaccent('%" . substr($escaped, 2, -1) . "%')"; + $escaped = $driver->escapeLike(substr($escaped, 2, -1),0); + $x[] = "public.unaccent($column) NOT ILIKE public.unaccent('%' || " . $escaped . " || '%')"; continue; } } From 37d738eeeeea7da1f6d67fe527c0dae00ee632c8 Mon Sep 17 00:00:00 2001 From: Nikita Golmgren Date: Tue, 11 Feb 2025 13:48:58 +0100 Subject: [PATCH 3/5] Fix ! excluding blanks --- src/DataSource/DibiFluentPostgreDataSource.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DataSource/DibiFluentPostgreDataSource.php b/src/DataSource/DibiFluentPostgreDataSource.php index daeb52fd9..f6e301cee 100755 --- a/src/DataSource/DibiFluentPostgreDataSource.php +++ b/src/DataSource/DibiFluentPostgreDataSource.php @@ -63,7 +63,7 @@ public function applyFilterText(Filter\FilterText $filter) if ($allow_negation_filter && strpos($word, Filter\FilterText::TOKEN_NEGATION) === 0) { //exclamation point means negation - the word is NOT included in the searched string $escaped = $driver->escapeLike(substr($escaped, 2, -1),0); - $x[] = "public.unaccent($column) NOT ILIKE public.unaccent('%' || " . $escaped . " || '%')"; + $x[] = "($column IS NULL OR $column = '' OR public.unaccent($column) NOT ILIKE public.unaccent('%' || " . $escaped . " || '%'))"; continue; } } From 5656a1e1d59a46737c416f753c72144c874b6dec Mon Sep 17 00:00:00 2001 From: Nikita Golmgren Date: Tue, 11 Feb 2025 14:53:21 +0100 Subject: [PATCH 4/5] #14102 allow filtering empty --- src/DataSource/DibiFluentPostgreDataSource.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/DataSource/DibiFluentPostgreDataSource.php b/src/DataSource/DibiFluentPostgreDataSource.php index f6e301cee..0dce37118 100755 --- a/src/DataSource/DibiFluentPostgreDataSource.php +++ b/src/DataSource/DibiFluentPostgreDataSource.php @@ -30,6 +30,9 @@ public function applyFilterText(Filter\FilterText $filter) if ($value === Filter\FilterText::TOKEN_EMPTY) { // Handle single '#' $this->data_source->where("($column IS NULL OR $column = '')"); continue; + } else if ($value === Filter\FilterText::TOKEN_NEGATION . Filter\FilterText::TOKEN_EMPTY) { + $this->data_source->where("($column IS NOT NULL AND $column <> '')"); + continue; } $value = str_replace(Filter\FilterText::TOKEN_EMPTY_ESCAPED, Filter\FilterText::TOKEN_EMPTY, $value); } From 5b4a9e44f4d0035cbdcbfa5a2266b6a310463450 Mon Sep 17 00:00:00 2001 From: Nikita Golmgren Date: Wed, 21 May 2025 16:05:59 +0200 Subject: [PATCH 5/5] #14102 Fix - add support for array data source, fix negation filters being disjuncted instead of conjuncted --- src/DataSource/ArrayDataSource.php | 52 ++++++++++++++++++- .../DibiFluentPostgreDataSource.php | 6 ++- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/src/DataSource/ArrayDataSource.php b/src/DataSource/ArrayDataSource.php index 9f7973fa0..53edcb642 100644 --- a/src/DataSource/ArrayDataSource.php +++ b/src/DataSource/ArrayDataSource.php @@ -168,7 +168,18 @@ protected function applyFilter($row, Filter $filter) $condition = $filter->getCondition(); + $is_negation_search = false; foreach ($condition as $column => $value) { + if ($filter->isSpecialChars()) { + if ($value === FilterText::TOKEN_EMPTY) { // Handle single '#' + return empty($row[$column]); + } else if ($value === FilterText::TOKEN_NEGATION . FilterText::TOKEN_EMPTY) { + return !empty($row[$column]); + } else if ($value === FilterText::TOKEN_EMPTY_ESCAPED) { // Handle '\#' for searching literal '#' + $value = FilterText::TOKEN_EMPTY; + } + } + if ($filter instanceof FilterText && $filter->isExactSearch()) { return $row[$column] == $value; } @@ -182,11 +193,48 @@ protected function applyFilter($row, Filter $filter) $row_value = strtolower(Strings::toAscii($row[$column])); foreach ($words as $word) { - if (strpos($row_value, strtolower(Strings::toAscii($word))) !== false) { - return $row; + if ($filter instanceof FilterText && $filter->isSpecialChars()) { + if ($word === FilterText::TOKEN_NEGATION . FilterText::TOKEN_EMPTY) { + return !empty($row_value); + } + $allow_negation_filter = true; + if (strpos($word, FilterText::TOKEN_NEGATION_ESCAPED) !== false) { + //If the escaped negation token is in the beginning of text, explicitly forbid parsing it after it's replaced + if (strpos($word, FilterText::TOKEN_NEGATION_ESCAPED) === 0) { + $allow_negation_filter = false; + } + + $word = str_replace(FilterText::TOKEN_NEGATION_ESCAPED, FilterText::TOKEN_NEGATION, $word); + } + + if ($allow_negation_filter && strpos($word, FilterText::TOKEN_NEGATION) === 0) { + //exclamation point means negation - the word is NOT included in the searched string + $excludedWord = substr($word, 1); + if (empty($excludedWord)) { + return true; + } + + $is_negation_search = true; + if (strpos($row_value, strtolower(Strings::toAscii($excludedWord))) !== false) { + return false; + } + } + + if (!$is_negation_search) { + return strpos($row_value, strtolower(Strings::toAscii($word))) !== false; + } + } else { + if (strpos($row_value, strtolower(Strings::toAscii($word))) !== false) { + return true; + } } } } + if ($is_negation_search) { + //we're looking for rows that don't have specific rows, and we haven't aborted by this point + //-> should be good + return true; + } } return false; diff --git a/src/DataSource/DibiFluentPostgreDataSource.php b/src/DataSource/DibiFluentPostgreDataSource.php index 0dce37118..81d1d2028 100755 --- a/src/DataSource/DibiFluentPostgreDataSource.php +++ b/src/DataSource/DibiFluentPostgreDataSource.php @@ -25,6 +25,7 @@ public function applyFilterText(Filter\FilterText $filter) $driver = $this->data_source->getConnection()->getDriver(); $or = []; + $is_negation_search = false; foreach ($condition as $column => $value) { if ($filter->isSpecialChars()) { if ($value === Filter\FilterText::TOKEN_EMPTY) { // Handle single '#' @@ -76,7 +77,10 @@ public function applyFilterText(Filter\FilterText $filter) $or[] = "((" . implode(") AND (", $x) . "))"; } - if (sizeof($or) > 1) { + if ($is_negation_search) { + $condition = sprintf("(%s)", implode(' AND ', $or)); + $this->data_source->where($condition); + } else if (sizeof($or) > 1) { $this->data_source->where('(%or)', $or); } else { $this->data_source->where($or);