Changeset 2414

Timestamp:

01/22/10 10:24:43 (2 years ago)

Author:

karpet

Message:

use Encoding::FixLatin? instead of our own code

Location:

Search-Tools/trunk

Files:

• Changes (modified) (1 diff)
• Makefile.PL (modified) (1 diff)
• Tools.xs (modified) (1 diff)
• lib/Search/Tools/Transliterate.pm (modified) (4 diffs)
• lib/Search/Tools/UTF8.pm (modified) (3 diffs)
• t/20transliterate-map.t (modified) (1 diff)
• t/28-byte-length.t (modified) (1 diff)

Search-Tools/trunk/Changes

@@ -218 +218 @@
     * add support for wildcard at start of term in addition to end of term.
     * added looks_like_win1252() to UTF8 class and convert1252() to Transliterate.
+    * added Encoding::FixLatin as a dependency.
     * fix off-by-one errors in find_bad_*_report and find_bad_* UTF8 functions.
-    * add debug_bytes_in_string() to UTF8 class.
+    * add debug_bytes() to UTF8 class.

Search-Tools/trunk/Makefile.PL

@@ -26 +26 @@
         'File::Slurp'         => 0,
         'Test::More'          => 0.94,
+        'Encoding::FixLatin'  => 0,
 
         #'Text::Aspell'          => '0.06',  # optional

Search-Tools/trunk/Tools.xs

@@ -131 +131 @@
 
 void
-debug_bytes_in_string(string)
+debug_bytes(string)
     SV* string;
 

Search-Tools/trunk/lib/Search/Tools/Transliterate.pm

@@ -6 +6 @@
 use Carp;
 use Encode;
+use Encoding::FixLatin qw( fix_latin );
 use Data::Dump qw( dump );
 
@@ -85 +86 @@
 B<0x80> and B<0x9f> inclusive. The 1252 codepoints are converted first to
 their UTF-8 counterparts per http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1252.TXT
-and then I<text> is run through convert().
+using Encoding::FixLatin and then I<text> is run through convert().
+
+Note that I<text> is checked with the looks_like_win1252() function from
+Search::Tools::UTF8 before calling fix_latin().
 
 =head1 BUGS
@@ -252 +256 @@
 }
 
-my %win1252 = (
-    "\x80" => "\x{20AC}",    #EURO SIGN
-
-    #\x81               #UNDEFINED
-    "\x82" => "\x{201A}",    #SINGLE LOW-9 QUOTATION MARK
-    "\x83" => "\x{0192}",    #LATIN SMALL LETTER F WITH HOOK
-    "\x84" => "\x{201E}",    #DOUBLE LOW-9 QUOTATION MARK
-    "\x85" => "\x{2026}",    #HORIZONTAL ELLIPSIS
-    "\x86" => "\x{2020}",    #DAGGER
-    "\x87" => "\x{2021}",    #DOUBLE DAGGER
-    "\x88" => "\x{02C6}",    #MODIFIER LETTER CIRCUMFLEX ACCENT
-    "\x89" => "\x{2030}",    #PER MILLE SIGN
-    "\x8A" => "\x{0160}",    #LATIN CAPITAL LETTER S WITH CARON
-    "\x8B" => "\x{2039}",    #SINGLE LEFT-POINTING ANGLE QUOTATION MARK
-    "\x8C" => "\x{0152}",    #LATIN CAPITAL LIGATURE OE
-
-    #\x8D               #UNDEFINED
-    "\x8E" => "\x{017D}",    #LATIN CAPITAL LETTER Z WITH CARON
-
-    #\x8F               #UNDEFINED
-    #\x90               #UNDEFINED
-    "\x91" => "\x{2018}",    #LEFT SINGLE QUOTATION MARK
-    "\x92" => "\x{2019}",    #RIGHT SINGLE QUOTATION MARK
-    "\x93" => "\x{201C}",    #LEFT DOUBLE QUOTATION MARK
-    "\x94" => "\x{201D}",    #RIGHT DOUBLE QUOTATION MARK
-    "\x95" => "\x{2022}",    #BULLET
-    "\x96" => "\x{2013}",    #EN DASH
-    "\x97" => "\x{2014}",    #EM DASH
-    "\x98" => "\x{02DC}",    #SMALL TILDE
-    "\x99" => "\x{2122}",    #TRADE MARK SIGN
-    "\x9A" => "\x{0161}",    #LATIN SMALL LETTER S WITH CARON
-    "\x9B" => "\x{203A}",    #SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
-    "\x9C" => "\x{0153}",    #LATIN SMALL LIGATURE OE
-
-    #\x9D               #UNDEFINED
-    "\x9E" => "\x{017E}",    #LATIN SMALL LETTER Z WITH CARON
-    "\x9F" => "\x{0178}",    #LATIN CAPITAL LETTER Y WITH DIAERESIS
-
-);
-
 sub convert1252 {
     my ( $self, $buf ) = @_;
-    my $newbuf = '';
 
     # don't bother unless we have non-ascii bytes
@@ -300 +263 @@
 
     $self->debug and warn "converting $buf\n";
-    while ( $buf =~ m/(.)/gox ) {
-        my $char = $1;
-        $self->debug and warn "$char\n";
-        if ( is_ascii($char) ) {
-            $self->debug and warn "$char is_ascii\n";
-            $newbuf .= $char;
-        }
-        elsif ( exists $win1252{$char} ) {
-            $self->debug and warn "$char is win1252\n";
-            $newbuf .= $win1252{$char};
-        }
-        else {
-            $self->debug and warn "append $char\n";
-            $newbuf .= $char;
-        }
-
-    }
-
+    my $newbuf = looks_like_win1252($buf) ? fix_latin($buf) : $buf;
     return $self->convert($newbuf);
 }

Search-Tools/trunk/lib/Search/Tools/UTF8.pm

@@ -20 +20 @@
     byte_length
     looks_like_win1252
-    debug_bytes_in_string
+    debug_bytes
 );
 
@@ -134 +134 @@
     if (   !is_latin1( $_[0] )
         && !is_ascii( $_[0] )
-        && $_[0] =~ m/[\x80-\x9f]/ )
+        && $_[0] =~ m/[\x00-\x7f]?[\x80-\x9f][\x00-\x7f]?/ )
     {
         return 1;
@@ -264 +264 @@
 See also the Search::Tools::Transliterate convert1252() method.
 
-=head2 debug_bytes_in_string( I<text> )
+=head2 debug_bytes( I<text> )
 
 Iterates over each byte in I<text>, printing byte, hex and decimal values

Search-Tools/trunk/t/20transliterate-map.t

@@ -41 +41 @@
 
 #$tr->debug(1);
-my $win1252 = "\xcf\x80\x82\x83\x91\x92\x93\x94\x9f";
-my $utf8_not_1252 = to_utf8("\xcf");  # \xc3\x8f
-ok( looks_like_win1252($win1252), "looks_like_win1252" );
-ok( !looks_like_win1252($utf8_not_1252), "utf8 string !looks_like_win1252");
+my $win1252 = "a\x{80}b\x{82}c\x{83}d\x{91}e\x{92}f\x{93}g";
+my $utf8_not_1252 = to_utf8("\xcf");    # \xc3\x8f
+ok( looks_like_win1252($win1252),        "looks_like_win1252" );
+ok( !looks_like_win1252($utf8_not_1252), "utf8 string !looks_like_win1252" );
 ok( my $win1252_conv = $tr->convert1252($win1252), "convert1252" );
 
-#debug_bytes_in_string($win1252);
-#debug_bytes_in_string($utf8_not_1252);
+#diag("win1252");
+#debug_bytes($win1252_conv);
+
+#debug_bytes($utf8_not_1252);
 
 #diag( dump $win1252_conv );
-is( $win1252_conv, qq{IEUR'f''""Y}, "transliterate 1252" );
+is( $win1252_conv, qq{aEURb'cfd'e'f"g}, "transliterate 1252" );
 
 1;

Search-Tools/trunk/t/28-byte-length.t

@@ -30 +30 @@
 
 diag("astr: $astr");
-debug_bytes_in_string($astr);
+debug_bytes($astr);
 
 diag("bstr: $bstr");
-debug_bytes_in_string($bstr);
+debug_bytes($bstr);