From a2d2e95555e98c88972052c449ea192aeca0eca6 Mon Sep 17 00:00:00 2001 From: Alexis Mousset Date: Sun, 10 Jan 2016 01:04:42 +0100 Subject: [PATCH] Use hashes to identify favicons --- controllers/bookmark.php | 2 +- controllers/feed.php | 2 +- controllers/history.php | 2 +- controllers/item.php | 2 +- lib/helpers.php | 4 +- models/favicon.php | 202 +++++++++++++++++++++++++++++ models/feed.php | 115 ++-------------- models/schema.php | 25 +++- vendor/composer/autoload_files.php | 1 + 9 files changed, 243 insertions(+), 112 deletions(-) create mode 100644 models/favicon.php diff --git a/controllers/bookmark.php b/controllers/bookmark.php index 622cf28..47f4607 100644 --- a/controllers/bookmark.php +++ b/controllers/bookmark.php @@ -41,7 +41,7 @@ Router\get_action('bookmarks', function() { $items = Model\Item\get_bookmarks($offset, Model\Config\get('items_per_page')); Response\html(Template\layout('bookmarks', array( - 'favicons' => Model\Feed\get_item_favicons($items), + 'favicons' => Model\Favicon\get_item_favicons($items), 'original_marks_read' => Model\Config\get('original_marks_read'), 'order' => '', 'direction' => '', diff --git a/controllers/feed.php b/controllers/feed.php index d636e43..fef46c2 100644 --- a/controllers/feed.php +++ b/controllers/feed.php @@ -124,7 +124,7 @@ Router\get_action('feeds', function() { } Response\html(Template\layout('feeds', array( - 'favicons' => Model\Feed\get_all_favicons(), + 'favicons' => Model\Favicon\get_all_favicons(), 'feeds' => Model\Feed\get_all_item_counts(), 'nothing_to_read' => $nothing_to_read, 'nb_unread_items' => $nb_unread_items, diff --git a/controllers/history.php b/controllers/history.php index 43eccd4..98e1957 100644 --- a/controllers/history.php +++ b/controllers/history.php @@ -15,7 +15,7 @@ Router\get_action('history', function() { ); Response\html(Template\layout('history', array( - 'favicons' => Model\Feed\get_item_favicons($items), + 'favicons' => Model\Favicon\get_item_favicons($items), 'original_marks_read' => Model\Config\get('original_marks_read'), 'items' => $items, 'order' => '', diff --git a/controllers/item.php b/controllers/item.php index b805148..e37f2b8 100644 --- a/controllers/item.php +++ b/controllers/item.php @@ -34,7 +34,7 @@ Router\get_action('unread', function() { } Response\html(Template\layout('unread_items', array( - 'favicons' => Model\Feed\get_item_favicons($items), + 'favicons' => Model\Favicon\get_item_favicons($items), 'original_marks_read' => Model\Config\get('original_marks_read'), 'order' => $order, 'direction' => $direction, diff --git a/lib/helpers.php b/lib/helpers.php index 68ad4b9..dfd34c3 100644 --- a/lib/helpers.php +++ b/lib/helpers.php @@ -42,7 +42,7 @@ function favicon_extension($type) 'image/jpg' => '.jpg' ); - if (in_array($type, $types)) { + if (array_key_exists($type, $types)) { return $types[$type]; } else { return '.ico'; @@ -52,7 +52,7 @@ function favicon_extension($type) function favicon(array $favicons, $feed_id) { if (! empty($favicons[$feed_id])) { - return ''; + return ''; } return ''; diff --git a/models/favicon.php b/models/favicon.php new file mode 100644 index 0000000..bf87fa0 --- /dev/null +++ b/models/favicon.php @@ -0,0 +1,202 @@ +getType(), $favicon->getContent()); + + if ($favicon_id === false) { + return false; + } + + return Database::getInstance('db') + ->table('favicons_feeds') + ->save(array( + 'feed_id' => $feed_id, + 'favicon_id' => $favicon_id + )); +} + +// Download a favicon +function fetch($feed_id, $site_url, $icon_link) +{ + if (Config\get('favicons') == 1 && ! has_favicon($feed_id)) { + $favicon = new Favicon; + $favicon->find($site_url, $icon_link); + return $favicon; + } + + return false; +} + +// Store the favicon (only if it does not exist yet) +function store($type, $icon) +{ + if ($icon == "") { + return false; + } + + $hash = sha1($icon); + + $favicon_id = get_favicon_id($hash); + + if ($favicon_id) { + return $favicon_id; + } + + $file = $hash.Helper\favicon_extension($type); + + if (file_put_contents(FAVICON_DIRECTORY.DIRECTORY_SEPARATOR.$file, $icon) === false) { + return false; + } + + $saved = Database::getInstance('db') + ->table('favicons') + ->save(array( + 'hash' => $hash, + 'type' => $type + )); + + if ($saved === false) { + return false; + } + + return get_favicon_id($hash); +} + +function get_favicon_id($hash) { + return Database::getInstance('db') + ->table('favicons') + ->eq('hash', $hash) + ->findOneColumn('id'); +} + +// Delete the favicon +function delete_favicon($favicon) +{ + unlink(FAVICON_DIRECTORY.DIRECTORY_SEPARATOR.$favicon['hash'].Helper\favicon_extension($favicon['type'])); + Database::getInstance('db') + ->table('favicons') + ->eq('hash', $favicon['hash']) + ->remove(); +} + +// Purge orphaned favicons from database +function purge_favicons() +{ + $favicons = Database::getInstance('db') + ->table('favicons') + ->columns( + 'favicons.type', + 'favicons.hash', + 'favicons_feeds.feed_id' + ) + ->join('favicons_feeds', 'favicon_id', 'id') + ->isnull('favicons_feeds.feed_id') + ->findAll(); + + foreach ($favicons as $favicon) { + delete_favicon($favicon); + } +} + +// Return true if the feed has a favicon +function has_favicon($feed_id) +{ + return Database::getInstance('db')->table('favicons_feeds')->eq('feed_id', $feed_id)->count() === 1; +} + +// Get favicons for those feeds +function get_favicons(array $feed_ids) +{ + if (Config\get('favicons') == 0) { + return array(); + } + + $result = array(); + + foreach ($feed_ids as $feed_id) { + $result[$feed_id] = Database::getInstance('db') + ->table('favicons') + ->columns( + 'favicons.type', + 'favicons.hash' + ) + ->join('favicons_feeds', 'favicon_id', 'id') + ->eq('favicons_feeds.feed_id', $feed_id) + ->findOne(); + } + + return $result; +} + +// Get all favicons for a list of items +function get_item_favicons(array $items) +{ + $feed_ids = array(); + + foreach ($items as $item) { + $feed_ids[$item['feed_id']] = $item['feed_id']; + } + + return get_favicons($feed_ids); +} + +// Get all favicons +function get_all_favicons() +{ + if (Config\get('favicons') == 0) { + return array(); + } + + $result = Database::getInstance('db') + ->table('favicons') + ->columns( + 'favicons_feeds.feed_id', + 'favicons.type', + 'favicons.hash' + ) + ->join('favicons_feeds', 'favicon_id', 'id') + ->findAll(); + + $map = array(); + + foreach ($result as $row) { + $map[$row['feed_id']] = array ( + "type" => $row['type'], + "hash" => $row['hash'] + ); + } + + return $map; +} + + diff --git a/models/feed.php b/models/feed.php index 7f84d3c..3e778b2 100644 --- a/models/feed.php +++ b/models/feed.php @@ -6,6 +6,7 @@ use UnexpectedValueException; use Model\Config; use Model\Item; use Model\Group; +use Model\Favicon; use Helper; use SimpleValidator\Validator; use SimpleValidator\Validators; @@ -13,109 +14,11 @@ use PicoDb\Database; use PicoFeed\Serialization\Export; use PicoFeed\Serialization\Import; use PicoFeed\Reader\Reader; -use PicoFeed\Reader\Favicon; use PicoFeed\PicoFeedException; use PicoFeed\Client\InvalidUrlException; const LIMIT_ALL = -1; -// Store the favicon -function store_favicon($feed_id, $link, $type, $icon) -{ - $file = $feed_id.Helper\favicon_extension($type); - - if (file_put_contents(FAVICON_DIRECTORY.DIRECTORY_SEPARATOR.$file, $icon) === false) { - return false; - } - - return Database::getInstance('db') - ->table('favicons') - ->save(array( - 'feed_id' => $feed_id, - 'link' => $link, - 'file' => $file, - 'type' => $type - )); -} - -// Delete the favicon -function delete_favicon($feed_id) -{ - foreach (get_favicons(array ($feed_id)) as $favicon) { - unlink(FAVICON_DIRECTORY.DIRECTORY_SEPARATOR.$favicon); - } -} - -// Delete all the favicons -function delete_all_favicons() -{ - foreach (get_all_favicons() as $favicon) { - unlink(FAVICON_DIRECTORY.DIRECTORY_SEPARATOR.$favicon); - } -} - -// Download favicon -function fetch_favicon($feed_id, $site_url, $icon_link) -{ - if (Config\get('favicons') == 1 && ! has_favicon($feed_id)) { - $favicon = new Favicon; - - $link = $favicon->find($site_url, $icon_link); - $icon = $favicon->getContent(); - $type = $favicon->getType(); - - if ($icon !== '') { - store_favicon($feed_id, $link, $type, $icon); - } - } -} - -// Return true if the feed have a favicon -function has_favicon($feed_id) -{ - return Database::getInstance('db')->table('favicons')->eq('feed_id', $feed_id)->count() === 1; -} - -// Get favicons for those feeds -function get_favicons(array $feed_ids) -{ - if (Config\get('favicons') == 0) { - return array(); - } - - $db = Database::getInstance('db') - ->hashtable('favicons') - ->columnKey('feed_id') - ->columnValue('file'); - - // pass $feeds_ids as argument list to hashtable::get(), use ... operator with php 5.6+ - return call_user_func_array(array($db, 'get'), $feed_ids); -} - -// Get all favicons for a list of items -function get_item_favicons(array $items) -{ - $feed_ids = array(); - - foreach ($items as $item) { - $feed_ids[$item['feed_id']] = $item['feed_id']; - } - - return get_favicons($feed_ids); -} - -// Get all favicons -function get_all_favicons() -{ - if (Config\get('favicons') == 0) { - return array(); - } - - return Database::getInstance('db') - ->hashtable('favicons') - ->getAll('feed_id', 'file'); -} - // Update feed information function update(array $values) { @@ -236,7 +139,7 @@ function create($url, $enable_grabber = false, $force_rtl = false, $cloak_referr Group\update_feed_groups($feed_id, $group_ids, $create_group); Item\update_all($feed_id, $feed->getItems()); - fetch_favicon($feed_id, $feed->getSiteUrl(), $feed->getIcon()); + Favicon\create_feed_favicon($feed_id, $feed->getSiteUrl(), $feed->getIcon()); } return $feed_id; @@ -301,7 +204,7 @@ function refresh($feed_id) update_cache($feed_id, $resource->getLastModified(), $resource->getEtag()); Item\update_all($feed_id, $feed->getItems()); - fetch_favicon($feed_id, $feed->getSiteUrl(), $feed->getIcon()); + Favicon\create_feed_favicon($feed_id, $feed->getSiteUrl(), $feed->getIcon()); } update_parsing_error($feed_id, 0); @@ -435,18 +338,20 @@ function update_cache($feed_id, $last_modified, $etag) // Remove one feed function remove($feed_id) { - delete_favicon($feed_id); Group\remove_all($feed_id); - + // Items are removed by a sql constraint - return Database::getInstance('db')->table('feeds')->eq('id', $feed_id)->remove(); + $result = Database::getInstance('db')->table('feeds')->eq('id', $feed_id)->remove(); + Favicon\purge_favicons(); + return $result; } // Remove all feeds function remove_all() { - delete_all_favicons(); - return Database::getInstance('db')->table('feeds')->remove(); + $result = Database::getInstance('db')->table('feeds')->remove(); + Favicon\purge_favicons(); + return $result; } // Enable a feed (activate refresh) diff --git a/models/schema.php b/models/schema.php index 42019c0..50f4dee 100644 --- a/models/schema.php +++ b/models/schema.php @@ -5,7 +5,30 @@ namespace Schema; use PDO; use Model\Config; -const VERSION = 42; +const VERSION = 43; + +function version_43(PDO $pdo) +{ + $pdo->exec('DROP TABLE favicons'); + + $pdo->exec( + 'CREATE TABLE favicons ( + id INTEGER PRIMARY KEY, + hash TEXT UNIQUE, + type TEXT + )' + ); + + $pdo->exec(' + CREATE TABLE "favicons_feeds" ( + feed_id INTEGER NOT NULL, + favicon_id INTEGER NOT NULL, + PRIMARY KEY(feed_id, favicon_id) + FOREIGN KEY(favicon_id) REFERENCES favicons(id) ON DELETE CASCADE + FOREIGN KEY(feed_id) REFERENCES feeds(id) ON DELETE CASCADE + ) + '); +} function version_42(PDO $pdo) { diff --git a/vendor/composer/autoload_files.php b/vendor/composer/autoload_files.php index e117b61..bf5858a 100644 --- a/vendor/composer/autoload_files.php +++ b/vendor/composer/autoload_files.php @@ -24,4 +24,5 @@ return array( $baseDir . '/models/database.php', $baseDir . '/models/remember_me.php', $baseDir . '/models/group.php', + $baseDir . '/models/favicon.php', );