--- old/usr/lib/ruby/1.8/samizdat/controllers/message_controller.rb~ 2008-07-23 01:34:49.000000000 +0200 +++ /usr/lib/ruby/1.8/samizdat/controllers/message_controller.rb 2008-08-16 02:42:16.000000000 +0200 @@ -9,3 +9,4 @@ # vim: et sw=2 sts=2 ts=8 tw=0 require 'samizdat/helpers/message_helper' +require 'samizdat/helpers/guess_lang' @@ -797,4 +798,5 @@ @message.content.title, message(@message, :full) << cut_warning.to_s << + lang_warning(body).to_s << '
' << _("Press 'Back' button to change the message.") << '
' << @@ -830,4 +832,24 @@ end end + def lang_warning(body) + if config['site']['language_guess'].to_s == 'warn' + old_lang = @request.language + apparent_lang = body.to_s.guess_lang + @request.language = @message.lang + message_lang = _('English') # what the selected language calls itself + @request.language = apparent_lang + + log(sprintf("m.1 @request.language = %s apparent_lang = %s @message.lang = %s\n", + @request.language,apparent_lang,@message.lang)) # DEBUG ONLY # + + if ( (body.kind_of? String) and @message.lang.guessable_lang and + ( apparent_lang != @message.lang) ) + warning = ''+sprintf(_('Warning: you said that your text is in %s, but it seems to be in another language.'), message_lang) + '
' + end + @request.language = old_lang + end + warning.to_s # '' if nil + end + end --- /dev/null 2008-08-15 00:06:02.612589253 +0200 +++ /usr/lib/ruby/1.8/samizdat/helpers/guess_lang.rb 2008-08-16 02:43:56.000000000 +0200 @@ -0,0 +1,66 @@ +# This is an *extremely* basic language detection algorithm. Three +# successfully found tokens are considered to signify a probable +# language detection. +# +# TODO: For a much more sophisticated algorithm with an extremely high +# accuracy, try porting libtextcat to ruby. +# WARNING: This is coded in UTF-8, not ascii. +# WARNING: Some scripts (e.g. latin) use spaces between words: do not +# remove these unless you're sure you know what you're doing. +# WARNING: If you break the functionality here, the worst that should +# happen is either too many or too few incorrect guesses. + + +class String + def guess_lang + length = self.length + count = {} + # ja + if length > 50 + count['ja']=0 + self.strip[0,1999].scan(%r{です|ます|あ|い|う|え|お|か|き|く|け|こ|さ|し|す|せ|そ|た|ち|つ|て|と|な|に|ぬ|ね|の|は|ひ|ふ|へ|ほ|ま|み|む|め|も|や|を|ア|イ|ウ|エ|オ|カ|キ|ク|ケ|コ|サ|シ|ス|セ|ソ|タ|チ|ツ|テ|ト|ナ|ニ|ヌ|ネ|ノ|ハ|ヒ|フ|ヘ|ホ|マ|ミ|ム|メ|モ|ヤ|ヲ}){ |s| count['ja'] +=1 } + count['ja']= count['ja'] * 2000/(length) if (0 < length and length < 2000) + end + if length > 100 + # en + count['en']=0 + self.strip[0,999].scan(%r{ a | the | of | in | this | that | which | and }){ |s| count['en'] +=1 } + count['en']= count['en'] * 1000/(length) if (0 < length and length < 1000) + # fr + count['fr']=0 + log(sprintf("0,999 = %s\n",self.strip[0,999])) # DEBUG ONLY # + self.strip[0,999].scan(%r{é|ê| à |ç|è| une | le | la | du | de la | dont | un |[^ ]on[st] }){ |s| + count['fr'] +=1 + log(sprintf("found: %s\n",s)) # DEBUG ONLY + log(sprintf("A count: %s\n",count['fr'])) # DEBUG ONLY + } + count['fr']= count['fr'] * 1000/(length) if (0 < length and length < 1000) + log(sprintf("B count: %s\n",count['fr'])) # DEBUG ONLY + # pl + count['pl']=0 + self.strip[0,999].scan(%r{ą|ć|ę|ł|ń|ó|ś|ż|ź|Ą|Ć|Ę|Ł|Ń|Ó|Ś|Ż|Ź| w | z | ze | od | czy | są | się |cz}){ |s| count['pl'] +=1 } + count['pl']= count['pl'] * 1000/(length) if (0 < length and length < 1000) + end + log(sprintf("C.1 count: %s\n",count.to_s)) # DEBUG ONLY + count = count.to_a.sort{|a,b| b[1] <=> a[1]} + log(sprintf("C.2 count: %s\n",count.to_s)) # DEBUG ONLY + + lang = 'und' # undetermined: http://en.wikipedia.org/wiki/List_of_ISO_639-2_codes + if count.length > 0 + lang = count[0][0] if count[0][1] > 3 + end + log(sprintf("D lang: %s\n",lang)) # DEBUG ONLY + lang + end + + def guessable_lang + case self + when 'en' then true + when 'fr' then true + when 'ja' then true + when 'pl' then true + else false + end + end + +end