Changeset 17749

Timestamp:

Sep 4, 2012, 10:04:34 PM (12 years ago)

Author:

rvelices

Message:

merge -r17748 from trunk to branch 2.4 bug 2735: fix/improve non latin language tags

1. non latin tags (greek/cyrillic...) are not sorted case-insesitive and group by letter view in tag list is not case insesitive
2. quick searching tag names does not perform correctly accent folding (e.g. Köln and Koln do not match) and case insesitivity for non latin letters
3. missing from remove_accents characters in romanian language (Latin Extended-B) ? c8 98 = LATIN CAPITAL LETTER S WITH COMMA BELOW ? c8 99 = LATIN SMALL LETTER S WITH COMMA BELOW ? c8 9a = LATIN CAPITAL LETTER T WITH COMMA BELOW ? c8 9b = LATIN SMALL LETTER T WITH COMMA BELOW
4. str2url allow non latin letters in output only if the input does not contain any valid lating letter/digit. we should always allow non latin letters in output

Location:

branches/2.4

Files:

• include/functions.inc.php (modified) (6 diffs)
• include/functions_html.inc.php (modified) (1 diff)
• include/functions_metadata.inc.php (modified) (1 diff)
• include/functions_search.inc.php (modified) (1 diff)
• tags.php (modified) (1 diff)

branches/2.4/include/functions.inc.php

@@ -204 +204 @@
  * @param string Str
  */
-function seems_utf8($Str) { # by bmorel at ssi dot fr
+function seems_utf8($Str) {
+  // OBSOLETE !!!
+  return qualify_utf8($Str) >= 0;
+}
+
+/* returns 0 if $str is Ascii, 1 if utf-8, -1 otherwise */
+function qualify_utf8($Str)
+{
+  $ret = 0;
   for ($i=0; $i<strlen($Str); $i++) {
     if (ord($Str[$i]) < 0x80) continue; # 0bbbbbbb
-    elseif ((ord($Str[$i]) & 0xE0) == 0xC0) $n=1; # 110bbbbb
+    $ret = 1;
+    if ((ord($Str[$i]) & 0xE0) == 0xC0) $n=1; # 110bbbbb
     elseif ((ord($Str[$i]) & 0xF0) == 0xE0) $n=2; # 1110bbbb
     elseif ((ord($Str[$i]) & 0xF8) == 0xF0) $n=3; # 11110bbb
     elseif ((ord($Str[$i]) & 0xFC) == 0xF8) $n=4; # 111110bb
     elseif ((ord($Str[$i]) & 0xFE) == 0xFC) $n=5; # 1111110b
-    else return false; # Does not match any model
+    else return -1; # Does not match any model
     for ($j=0; $j<$n; $j++) { # n bytes matching 10bbbbbb follow ?
       if ((++$i == strlen($Str)) || ((ord($Str[$i]) & 0xC0) != 0x80))
-      return false;
-    }
-  }
-  return true;
+        return -1;
+    }
+  }
+  return $ret;
 }
 
@@ -226 +235 @@
 function remove_accents($string)
 {
-  if ( !preg_match('/[\x80-\xff]/', $string) )
-    return $string;
-
-  if (seems_utf8($string)) {
+  $utf = qualify_utf8($string);
+  if ( $utf == 0 )
+    return $string; // ascii
+
+  if ( $utf > 0 ) {
     $chars = array(
     // Decompositions for Latin-1 Supplement
@@ -324 +334 @@
     "\xc5\xbc"=>'z', "\xc5\xbd"=>'Z',
     "\xc5\xbe"=>'z', "\xc5\xbf"=>'s',
+    // Decompositions for Latin Extended-B
+    "\xc8\x98"=>'S', "\xc8\x99"=>'s',
+    "\xc8\x9a"=>'T', "\xc8\x9b"=>'t',
     // Euro Sign
     "\xe2\x82\xac"=>'E',
@@ -354 +367 @@
 }
 
+if (function_exists('mb_strtolower') && defined('PWG_CHARSET'))
+{
+  function transliterate($term)
+  {
+    return remove_accents( mb_strtolower($term, PWG_CHARSET) );
+  }
+}
+else
+{
+  function transliterate($term)
+  {
+    return remove_accents( strtolower($term) );
+  }
+}
+
+
+
 /**
  * simplify a string to insert it into an URL
@@ -362 +392 @@
 function str2url($str)
 {
-  $raw = $str;
-
-  $str = remove_accents($str);
-  $str = preg_replace('/[^a-z0-9_\s\'\:\/\[\],-]/','',strtolower($str));
+  $str = $safe = transliterate($str);
+  $str = preg_replace('/[^\x80-\xffa-z0-9_\s\'\:\/\[\],-]/','',$str);
   $str = preg_replace('/[\s\'\:\/\[\],-]+/',' ',trim($str));
   $res = str_replace(' ','_',$str);
@@ -371 +399 @@
   if (empty($res))
   {
-    $res = str_replace(' ','_', $raw);
+    $res = str_replace(' ','_', $safe);
   }
 

branches/2.4/include/functions_html.inc.php

@@ -301 +301 @@
     if (!isset($cache[__FUNCTION__][ $tag['name'] ]))
     {
-      $cache[__FUNCTION__][ $tag['name'] ] = strtolower(str2url($tag['name']));
+      $cache[__FUNCTION__][ $tag['name'] ] = transliterate($tag['name']);
     }
   }

branches/2.4/include/functions_metadata.inc.php

@@ -91 +91 @@
     // how to detect it so a plugin should do the trick.
     $value = trigger_event('clean_iptc_value', $value);
-    $is_utf8 = seems_utf8($value);
-    $value = convert_charset( $value,
-      $is_utf8 ? 'utf-8' : 'iso-8859-1',
-      get_pwg_charset() );
+    if ( ($qual = qualify_utf8($value)) != 0)
+    {// has non ascii chars
+      $value = convert_charset( $value,
+        $qual>0 ? 'utf-8' : 'iso-8859-1',
+        get_pwg_charset() );
+    }
   }
   return $value;

branches/2.4/include/functions_search.inc.php

@@ -267 +267 @@
 
 
-if (function_exists('mb_strtolower'))
-{
-  function transliterate($term)
-  {
-    return remove_accents( mb_strtolower($term) );
-  }
-}
-else
-{
-  function transliterate($term)
-  {
-    return remove_accents( strtolower($term) );
-  }
-}
-
 function is_word_char($ch)
 {

branches/2.4/tags.php

@@ -100 +100 @@
   foreach ($tags as $tag)
   {
-    $tag_letter = strtoupper(mb_substr(str2url($tag['name']), 0, 1, 'utf-8'));
+    $tag_letter = mb_strtoupper(mb_substr(transliterate($tag['name']), 0, 1, PWG_CHARSET), PWG_CHARSET);
 
     if ($current_tag_idx==0) {