--- s080703/samizdat/lib/samizdat/helpers/syndication_helper.rb 2008-05-10 18:35:32.000000000 +0200
+++ /usr/lib/ruby/1.8/samizdat/helpers/syndication_helper.rb 2008-07-19 18:42:05.000000000 +0200
@@ -13,7 +13,7 @@
module SyndicationHelper
def each_import_feed
- feeds = config['import_feeds']
+ feeds = (config['import_feeds'] or {})['feeds']
if feeds.kind_of? Hash
feeds.keys.sort.each {|name| yield name, feeds[name] }
end
@@ -35,7 +35,14 @@
end
next unless url.kind_of? String
- feed = persistent_cache[ '/import_feeds/' + url ] # first slash to avoid clashes with site_name
+ # find the cached feed in the most preferred language
+ feed = nil
+ @request.accept_language.each do |l|
+ # first slash to avoid clashes with site_name
+ feed = persistent_cache[ '/import_feeds/' +
+ %{#{l}/} + url.to_s ]
+ break unless feed.nil?
+ end
if feed.kind_of? Array and feed.size > 0
output << %{
#{feed_name}
\n
}
--- s080703/samizdat/bin/samizdat-import-feeds 2008-05-10 18:35:32.000000000 +0200
+++ /usr/bin/samizdat-import-feeds 2008-07-19 18:24:27.000000000 +0200
@@ -47,6 +47,7 @@
#
def run
feeds = {}
+ @feed_languages = {} # a hash of arrays
@sites.each do |site_name|
ENV['SAMIZDAT_SITE'] = site_name
@@ -59,10 +60,25 @@
options
end
- # fetch once per uri across all sites
- (feeds[url] ||= []).push(site_name) if url
+ # fetch once per uri and language across all sites
+ if url
+ (feeds[url] ||= []).push(site_name)
+
+ if @feed_languages[url]
+ @feed_languages[url] +=
+ ((config['import_feeds'] or {})['languages'] or
+ [default_language])
+ else
+ @feed_languages[url] =
+ ((config['import_feeds'] or {})['languages'] or
+ [default_language])
end
end
+ end # each_import_feed do |feed_name, options|
+ end # @sites.each do |site_name|
+
+ @feed_languages.each { |url, languages|
+ languages.sort!.uniq! }
feeds.each do |url, sites|
begin
@@ -80,32 +96,64 @@
def update_feed(url, sites)
response = nil # scope fix
+ failed_urls = []
+ failed_parses= []
+
+ @feed_languages[url] and @feed_languages[url].each do |language|
+ begin # try a URL
+ # hack for non-(language-negotiating) feeds: substitute LANG
+ # in the URL with desired language
+ if url =~ %r{(//[^/]+/.*)LANG} # LANG must be in relative part of URL
+ try_url = url.sub( %r{(//[^/]+/.*)LANG}, '\1' +%{#{language}} )
+ else
+ try_url= url
+ end
+ response = nil
begin
Timeout.timeout(CONNECTION_TIMEOUT) do
- response = open(url, "Accept-Language" => default_language) {|file| file.read }
+ response = open(try_url, "Accept-Language" =>
+ language) {|file| file.read }
end
- rescue => error
- raise FeedUpdaterError, "Failed to fetch feed from #{url}: " + error.message
+ rescue
+ failed_urls += [try_url]
end
+ end # try a URL
begin
feed = parse_feed(response)
- rescue => error
- raise FeedUpdaterError, "Failed to parse feed from #{url}: " + error.message
+ ok = true
+ rescue
+ failed_parses += [try_url]
+ ok = false
+ end
+
+ if ok
+# sites.each do |site_name| # loop seems unnecessary for persistent_cache
+# ENV['SAMIZDAT_SITE'] = site_name
+ # first slash to avoid clashes with site_name
+ persistent_cache['/import_feeds/' + %{#{language}/} + url.to_s ] = feed
+# end
end
- sites.each do |site_name|
- ENV['SAMIZDAT_SITE'] = site_name
- persistent_cache['/import_feeds/' + url] = feed # first slash to avoid clashes with site_name
+ end # @feed_languages[url].each do |language|
+
+ if failed_urls.size > 0
+ raise FeedUpdaterError,
+ sprintf("Failed to fetch feed from:\n %s", failed_urls.each{|u| u + '\n' })
+ end
+ if failed_parses.size > 0
+ raise FeedUpdaterError,
+ sprintf("Failed to parse feed from:\n %s", failed_parses.each{|u| u + '\n' })
end
end
+
def parse_feed(response)
- # Remove tag section not needed and known to be buggy for invalid "mn" type
- # URI http://usefulinc.com/rss/manifest/
- #
- # fixme: explain this better
- #
+ # Some existing feeds (sf-active, 2007) refer to a name space
+ # defined at the non-existent URI:
+ # xmlns:mn="http://usefulinc.com/rss/manifest/" They also provide a
+ # tag using this namespace. This can cause a parsing failure.
+ # Here, we heuristically remove this undefinable tag.
if response =~ %r{http://usefulinc.com/rss/manifest/}
response.sub!(%r{}, '')
end
--- s080703/samizdat/data/samizdat/config.yaml 2008-05-10 18:35:32.000000000 +0200
+++ config.yaml 2008-07-19 18:33:15.000000000 +0200
@@ -120,15 +120,32 @@
# good idea to put it in cron).
#
#import_feeds:
+# # Languages to be used in content-negotiation or to
+# # to be subtituted into 'LANG' in the URLs. Default
+# # is to only use the first preferred language above.
+# 'languages': [ en, fr ]
+#
+# 'feeds':
#
# # Display name of the feed
# 'Local Samizdat':
#
-# # Feed URL
+# # Feed URL with language content-negotation
# url: 'http://localhost/samizdat/frontpage/rss'
#
# # Number of entries to display (no bigger than limit:page)
# limit: 5
+#
+# # Display name of the feed
+# 'Local CMS':
+#
+# # Feed URLs with language hardcoded in URL:
+# # real URLs: http://localhost/some_cms/en/frontpage
+# # http://localhost/some_cms/fr/frontpage
+# url: 'http://localhost/some_cms/LANG/frontpage'
+#
+# # Number of entries to display (no bigger than limit:page)
+# limit: 3
# Database Connection
#