Changeset 2561

Timestamp:

03/02/10 00:08:44 (2 years ago)

Author:

karpet

Message:

add analyzer per-field

Location:

Search-Query-Dialect-KSx/trunk

Files:

• lib/Search/Query/Dialect/KSx.pm (modified) (5 diffs)
• lib/Search/Query/Field/KSx.pm (modified) (3 diffs)
• t/01-parser.t (modified) (2 diffs)

Search-Query-Dialect-KSx/trunk/lib/Search/Query/Dialect/KSx.pm

@@ -5 +5 @@
 use Carp;
 use Data::Dump qw( dump );
+use Scalar::Util qw( blessed );
 use Search::Query::Field::KSx;
 use KinoSearch::Search::ANDQuery;
@@ -122 +123 @@
 
     if ( $self->fuzzify ) {
-        $value .= '*' unless $value =~ m/[\*\%]/;
-    }
-
-    # normalize wildcard
-    my $wildcard = $self->wildcard;
-    $value =~ s/[\*\%]/$wildcard/g;
+        $value .= '*' unless $value =~ m/[\*]/;
+    }
 
     return $value;
@@ -350 +347 @@
 
     my $quote = $clause->quote || '';
+    my $is_phrase = $quote eq '"' ? 1 : 0;
 
     my @buf;
-NAME: for my $name (@fields) {
+FIELD: for my $name (@fields) {
         my $field = $self->_get_field($name);
 
         if ( defined $field->callback ) {
             push( @buf, $field->callback->( $field, $op, $value ) );
-            next NAME;
+            next FIELD;
         }
 
         #warn dump [ $name, $op, $quote, $value ];
 
-        # invert fuzzy
-        if ( $op eq '!~' || ( $op eq '!:' and $value =~ m/[$wildcard\*\?]/ ) )
-        {
-            $value .= $wildcard unless $value =~ m/\Q$wildcard/;
-
-            push(
-                @buf,
-                Search::Query::Dialect::KSx::NOTWildcardQuery->new(
-                    field => $name,
-                    term  => $value,
-                )
-            );
-        }
-
-        # fuzzy
-        elsif ( $op eq '~'
-            || ( $op eq ':' and $value =~ m/[$wildcard\*\?]/ ) )
-        {
-            $value .= $wildcard unless $value =~ m/\Q$wildcard/;
-
-            push(
-                @buf,
-                Search::Query::Dialect::KSx::WildcardQuery->new(
-                    field => $name,
-                    term  => $value,
-                )
-            );
-        }
-
-        # invert
-        elsif ( $op eq '!:' ) {
-            push(
-                @buf,
-                KinoSearch::Search::NOTQuery->new(
-                    field => $name,
-                    term  => $value,
-                )
-            );
-        }
-
-        # range
-        elsif ( $op eq '..' ) {
+        # range is un-analyzed
+        if ( $op eq '..' ) {
             if ( ref $value ne 'ARRAY' or @$value != 2 ) {
                 croak "range of values must be a 2-element ARRAY";
@@ -417 +375 @@
 
             push( @buf, $range_query );
+            next FIELD;
 
         }
@@ -427 +386 @@
 
             croak "NOT Range query not yet supported";
-        }
-
-        # standard
-        else {
+            next FIELD;    # haha. never get here.
+        }
+
+        $self->debug and warn "value before:$value";
+        my @values = ($value);
+
+        # if the field has an analyzer, use it on $value
+        if ( blessed( $field->analyzer ) && !ref $value ) {
+
+            # preserve any wildcards
+            if ( $value =~ m/[$wildcard\*\?]/ ) {
+
+                # assume CaseFolder
+                $value = lc($value);
+
+                # split on whitespace, not token regex
+                my @tok = split( m/\s+/, $value );
+
+                # if stemmer, apply only to prefix if at all.
+                my $stemmer;
+                if ($field->analyzer->isa(
+                        'KinoSearch::Analysis::PolyAnalyzer')
+                    )
+                {
+
+    # KS currently broken with no get_analyzers() method.
+    #                    my $analyzers = $field->analyzer->get_analyzers();
+    #                    for my $ana (@$analyzers) {
+    #                        if (   $ana->isa('KinoSearch::Analysis::Stemmer')
+    #                            or $ana->can('stem') )
+    #                        {
+    #                            $stemmer = $ana;
+    #                            last;
+    #                        }
+    #                    }
+                }
+                elsif ($field->analyzer->isa('KinoSearch::Analysis::Stemmer')
+                    or $field->analyzer->can('stem') )
+                {
+                    $stemmer = $field->analyzer;
+                }
+
+                if ($stemmer) {
+                    carp "found stemmer";
+                    for my $tok (@tok) {
+                        if ( $tok =~ m/^\w\*$/ ) {
+                            $tok = $stemmer->stem($tok);
+                        }
+                    }
+                }
+
+            }
+            else {
+                @values = grep { defined and length }
+                    @{ $field->analyzer->split($value) };
+            }
+        }
+
+        $self->debug and warn "value after :" . dump( \@values );
+
+        if ( $is_phrase or @values > 1 ) {
             push(
                 @buf,
-                KinoSearch::Search::TermQuery->new(
+                KinoSearch::Search::PhraseQuery->new(
                     field => $name,
-                    term  => $value,
+                    terms => \@values,
                 )
             );
         }
+        else {
+            my $term = $values[0];
+
+            # invert fuzzy
+            if ( $op eq '!~'
+                || ( $op eq '!:' and $term =~ m/[$wildcard\*\?]/ ) )
+            {
+                $term .= $wildcard unless $term =~ m/\Q$wildcard/;
+
+                push(
+                    @buf,
+                    Search::Query::Dialect::KSx::NOTWildcardQuery->new(
+                        field => $name,
+                        term  => $term,
+                    )
+                );
+            }
+
+            # fuzzy
+            elsif ( $op eq '~'
+                || ( $op eq ':' and $term =~ m/[$wildcard\*\?]/ ) )
+            {
+                $term .= $wildcard unless $term =~ m/\Q$wildcard/;
+
+                push(
+                    @buf,
+                    Search::Query::Dialect::KSx::WildcardQuery->new(
+                        field => $name,
+                        term  => $term,
+                    )
+                );
+            }
+
+            # invert
+            elsif ( $op eq '!:' ) {
+                push(
+                    @buf,
+                    KinoSearch::Search::NOTQuery->new(
+                        field => $name,
+                        term  => $term,
+                    )
+                );
+            }
+
+            # standard
+            else {
+                push(
+                    @buf,
+                    KinoSearch::Search::TermQuery->new(
+                        field => $name,
+                        term  => $term,
+                    )
+                );
+            }
+
+        }    # TERM
     }
     if ( @buf == 1 ) {

Search-Query-Dialect-KSx/trunk/lib/Search/Query/Field/KSx.pm

@@ -3 +3 @@
 use warnings;
 use base qw( Search::Query::Field );
+use Scalar::Util qw( blessed );
 
-__PACKAGE__->mk_accessors(qw( type is_int ));
+__PACKAGE__->mk_accessors(qw( type is_int analyzer ));
 
 our $VERSION = '0.01';
@@ -37 +38 @@
 =item type
 
-The column type.a
+The column type. This may be a KinoSearch::FieldType object
+or a simple string.
 
 =item is_int
 
 Set if C<type> matches m/int|num|date/.
+
+=item analyzer
+
+Set to a KinoSearch::Analysis::Analyzer-based object (optional).
 
 =back
@@ -54 +60 @@
 
     # numeric types
-    if ( $self->{type} =~ m/int|date|num/ ) {
+    if ( !blessed( $self->{type} ) && $self->{type} =~ m/int|date|num/ ) {
         $self->{is_int} = 1;
     }

Search-Query-Dialect-KSx/trunk/t/01-parser.t

@@ -3 +3 @@
 use strict;
 use warnings;
-use Test::More tests => 53;
+use Test::More tests => 54;
 use Data::Dump qw( dump );
 
+use KinoSearch::Analysis::PolyAnalyzer;
+my $analyzer = KinoSearch::Analysis::PolyAnalyzer->new( language => 'en', );
+
 use_ok('Search::Query::Parser');
 
 ok( my $parser = Search::Query::Parser->new(
-        fields         => [qw( foo color name )],
+        fields => {
+            foo   => { analyzer => $analyzer },
+            color => { analyzer => $analyzer },
+            name  => { analyzer => $analyzer },
+        },
         default_field  => 'name',
         dialect        => 'KSx',
@@ -19 +26 @@
 #dump $parser;
 
-ok( my $query1 = $parser->parse('foo=bar'), "query1" );
-
-is( $query1, qq/foo:bar/, "query1 string" );
+ok( my $query1 = $parser->parse('foo=BAR'), "query1" );
+
+is( $query1, qq/foo:BAR/, "query1 string" );
 
 ok( my $ks_query1 = $query1->as_ks_query(), "as_ks_query" );
 ok( $ks_query1->isa('KinoSearch::Search::TermQuery'),
     "ks_query isa TermQuery" );
-
-ok( my $query2 = $parser->parse('foo:bar'), "query2" );
-
-is( $query2, qq/foo:bar/, "query2 string" );
-
-ok( my $query3 = $parser->parse('foo bar'), "query3" );
-
-is( $query3, qq/name:foo AND name:bar/, "query3 string" );
+is( $ks_query1->to_string, "foo:bar", "KS query analyzer applied" );
+
+ok( my $query2 = $parser->parse('foo:BaR'), "query2" );
+
+is( $query2, qq/foo:BaR/, "query2 string" );
+
+ok( my $query3 = $parser->parse('FoO bar'), "query3" );
+
+is( $query3, qq/name:FoO AND name:bar/, "query3 string" );
 
 my $str = '-color:red (name:john OR foo:bar)';