jan
/
miniflux-legacy
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add support for Expires and Cache-Control headers

This commit is contained in:
Frederic Guillot 2016-12-28 20:24:00 -05:00
parent 2a9fd5fb62
commit fe614579ac
13 changed files with 255 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ChangeLog
View File

@ -13,6 +13,7 @@ Version 1.2.0 (unreleased)
* Add support for Wallabag service
* Show last parsing error message in user interface
* Disable automatically a feed after too many failures
* Add support for Expires and Cache-Control headers (HTTP cache)
* Add unit tests
Migration procedure from 1.1.x to 1.2.0:

4
app/handlers/feed.php
View File

@ -71,6 +71,7 @@ function create_feed($user_id, $url, $download_content = false, $rtl = false, $c
$feed,
$resource->getEtag(),
$resource->getLastModified(),
$resource->getExpiration()->getTimestamp(),
$rtl,
$download_content,
$cloak_referrer
@ -120,6 +121,7 @@ function update_feed($user_id, $feed_id)
'etag' => $resource->getEtag(),
'last_modified' => $resource->getLastModified(),
'last_checked' => time(),
'expiration' => $resource->getExpiration()->getTimestamp(),
'parsing_error' => 0,
'parsing_error_message' => '',
));
@ -135,7 +137,7 @@ function update_feed($user_id, $feed_id)
function update_feeds($user_id, $limit = null)
{
foreach (Model\Feed\get_feed_ids($user_id, $limit) as $feed_id) {
foreach (Model\Feed\get_feed_ids_to_refresh($user_id, $limit) as $feed_id) {
update_feed($user_id, $feed_id);
}
}

17
app/models/feed.php
View File

@ -6,13 +6,14 @@ use Miniflux\Model\Favicon;
use Miniflux\Model\Item;
use Miniflux\Model\Group;
use PicoDb\Database;
use PicoDb\SQLException;
use PicoFeed\Parser\Feed;
const STATUS_ACTIVE = 1;
const STATUS_INACTIVE = 0;
const TABLE = 'feeds';
function create($user_id, Feed $feed, $etag, $last_modified, $rtl = false, $scraper = false, $cloak_referrer = false)
function create($user_id, Feed $feed, $etag, $last_modified, $expiration = 0, $rtl = false, $scraper = false, $cloak_referrer = false)
{
$db = Database::getInstance('db');
@ -32,6 +33,7 @@ function create($user_id, Feed $feed, $etag, $last_modified, $rtl = false, $scra
'etag' => $etag,
'last_modified' => $last_modified,
'last_checked' => time(),
'expiration' => $expiration,
'cloak_referrer' => $cloak_referrer ? 1 : 0,
));
@ -85,12 +87,17 @@ function get_feeds_with_items_count($user_id)
return $feeds;
}
function get_feed_ids($user_id, $limit = null)
function get_feed_ids_to_refresh($user_id, $limit = null, $expiration = 0)
{
if ($expiration === 0) {
$expiration = time();
}
$query = Database::getInstance('db')
->table(TABLE)
->eq('user_id', $user_id)
->eq('enabled', STATUS_ACTIVE)
->lte('expiration', $expiration)
->asc('last_checked')
->asc('id');
@ -113,6 +120,8 @@ function get_feed($user_id, $feed_id)
function update_feed($user_id, $feed_id, array $values)
{
$db = Database::getInstance('db');
try {
$db->startTransaction();
$feed = $values;
@ -124,7 +133,7 @@ function update_feed($user_id, $feed_id, array $values)
->table('feeds')
->eq('user_id', $user_id)
->eq('id', $feed_id)
->save($feed);
->update($feed);
if ($result) {
if (isset($values['feed_group_ids']) && isset($values['group_name']) &&
@ -137,6 +146,8 @@ function update_feed($user_id, $feed_id, array $values)
return true;
}
} catch (SQLException $e) {}
$db->cancelTransaction();
return false;
}

7
app/schemas/postgres.php
View File

@ -5,7 +5,12 @@ namespace Miniflux\Schema;
use PDO;
use Miniflux\Helper;
const VERSION = 2;
const VERSION = 3;
function version_3(PDO $pdo)
{
$pdo->exec('ALTER TABLE feeds ADD COLUMN expiration BIGINT DEFAULT 0');
}
function version_2(PDO $pdo)
{

7
app/schemas/sqlite.php
View File

@ -5,7 +5,12 @@ namespace Miniflux\Schema;
use PDO;
use Miniflux\Helper;
const VERSION = 2;
const VERSION = 3;
function version_3(PDO $pdo)
{
$pdo->exec('ALTER TABLE feeds ADD COLUMN expiration INTEGER DEFAULT 0');
}
function version_2(PDO $pdo)
{

2
composer.json
View File

@ -15,7 +15,7 @@
"fguillot/simple-validator": "v1.0.0",
"fguillot/json-rpc": "v1.2.3",
"fguillot/picodb": "v1.0.14 ",
"fguillot/picofeed": "v0.1.27",
"fguillot/picofeed": "v0.1.28",
"pda/pheanstalk": "v3.1.0",
"ircmaxell/password-compat": "^1.0.4"
},

2
producer.php
View File

@ -17,7 +17,7 @@ $limit = get_cli_option('limit', $options);
$connection = new Pheanstalk(BEANSTALKD_HOST);
foreach (Model\User\get_all_users() as $user) {
foreach (Model\Feed\get_feed_ids($user['id'], $limit) as $feed_id) {
foreach (Model\Feed\get_feed_ids_to_refresh($user['id'], $limit) as $feed_id) {
$payload = serialize(array(
'feed_id' => $feed_id,
'user_id' => $user['id'],

17
tests/unit/FeedModelTest.php
View File

@ -64,13 +64,22 @@ class FeedModelTest extends BaseTest
$feed->setTitle('Some feed');
$feed->setFeedUrl('another feed url');
$feed->setSiteUrl('site url');
$this->assertEquals(2, Model\Feed\create(1, $feed, 'etag', 'last modified'));
$this->assertEquals(2, Model\Feed\create(1, $feed, 'etag', 'last modified', time()));
$feed_ids = Model\Feed\get_feed_ids(1);
$this->assertEquals(array(1, 2), $feed_ids);
$feed = new Feed();
$feed->setTitle('Some feed');
$feed->setFeedUrl('some other feed url');
$feed->setSiteUrl('site url');
$this->assertEquals(3, Model\Feed\create(1, $feed, 'etag', 'last modified', strtotime('-1 week')));
$feed_ids = Model\Feed\get_feed_ids(1, 1);
$feed_ids = Model\Feed\get_feed_ids_to_refresh(1);
$this->assertEquals(array(1, 2, 3), $feed_ids);
$feed_ids = Model\Feed\get_feed_ids_to_refresh(1, 1);
$this->assertEquals(array(1), $feed_ids);
$feed_ids = Model\Feed\get_feed_ids_to_refresh(1, null, strtotime('-2 days'));
$this->assertEquals(array(1, 3), $feed_ids);
}
public function testGetFeedWithItemsCount()

2
vendor/autoload.php vendored
View File

@ -4,4 +4,4 @@
require_once __DIR__ . '/composer' . '/autoload_real.php';
return ComposerAutoloaderInitfd7e8d436e1dc450edc3153ac8bc31b4::getLoader();
return ComposerAutoloaderInitfc88bf644a3dc0d30b5792ad2fec7895::getLoader();

14
vendor/composer/autoload_real.php vendored
View File

@ -2,7 +2,7 @@
// autoload_real.php @generated by Composer
class ComposerAutoloaderInitfd7e8d436e1dc450edc3153ac8bc31b4
class ComposerAutoloaderInitfc88bf644a3dc0d30b5792ad2fec7895
{
private static $loader;
@ -19,15 +19,15 @@ class ComposerAutoloaderInitfd7e8d436e1dc450edc3153ac8bc31b4
return self::$loader;
}
spl_autoload_register(array('ComposerAutoloaderInitfd7e8d436e1dc450edc3153ac8bc31b4', 'loadClassLoader'), true, true);
spl_autoload_register(array('ComposerAutoloaderInitfc88bf644a3dc0d30b5792ad2fec7895', 'loadClassLoader'), true, true);
self::$loader = $loader = new \Composer\Autoload\ClassLoader();
spl_autoload_unregister(array('ComposerAutoloaderInitfd7e8d436e1dc450edc3153ac8bc31b4', 'loadClassLoader'));
spl_autoload_unregister(array('ComposerAutoloaderInitfc88bf644a3dc0d30b5792ad2fec7895', 'loadClassLoader'));
$useStaticLoader = PHP_VERSION_ID >= 50600 && !defined('HHVM_VERSION');
if ($useStaticLoader) {
require_once __DIR__ . '/autoload_static.php';
call_user_func(\Composer\Autoload\ComposerStaticInitfd7e8d436e1dc450edc3153ac8bc31b4::getInitializer($loader));
call_user_func(\Composer\Autoload\ComposerStaticInitfc88bf644a3dc0d30b5792ad2fec7895::getInitializer($loader));
} else {
$map = require __DIR__ . '/autoload_namespaces.php';
foreach ($map as $namespace => $path) {
@ -48,19 +48,19 @@ class ComposerAutoloaderInitfd7e8d436e1dc450edc3153ac8bc31b4
$loader->register(true);
if ($useStaticLoader) {
$includeFiles = Composer\Autoload\ComposerStaticInitfd7e8d436e1dc450edc3153ac8bc31b4::$files;
$includeFiles = Composer\Autoload\ComposerStaticInitfc88bf644a3dc0d30b5792ad2fec7895::$files;
} else {
$includeFiles = require __DIR__ . '/autoload_files.php';
}
foreach ($includeFiles as $fileIdentifier => $file) {
composerRequirefd7e8d436e1dc450edc3153ac8bc31b4($fileIdentifier, $file);
composerRequirefc88bf644a3dc0d30b5792ad2fec7895($fileIdentifier, $file);
}
return $loader;
}
}
function composerRequirefd7e8d436e1dc450edc3153ac8bc31b4($fileIdentifier, $file)
function composerRequirefc88bf644a3dc0d30b5792ad2fec7895($fileIdentifier, $file)
{
if (empty($GLOBALS['__composer_autoload_files'][$fileIdentifier])) {
require $file;

10
vendor/composer/autoload_static.php vendored
View File

@ -4,7 +4,7 @@
namespace Composer\Autoload;
class ComposerStaticInitfd7e8d436e1dc450edc3153ac8bc31b4
class ComposerStaticInitfc88bf644a3dc0d30b5792ad2fec7895
{
public static $files = array (
'e40631d46120a9c38ea139981f8dab26' => __DIR__ . '/..' . '/ircmaxell/password-compat/lib/password.php',
@ -277,10 +277,10 @@ class ComposerStaticInitfd7e8d436e1dc450edc3153ac8bc31b4
public static function getInitializer(ClassLoader $loader)
{
return \Closure::bind(function () use ($loader) {
$loader->prefixLengthsPsr4 = ComposerStaticInitfd7e8d436e1dc450edc3153ac8bc31b4::$prefixLengthsPsr4;
$loader->prefixDirsPsr4 = ComposerStaticInitfd7e8d436e1dc450edc3153ac8bc31b4::$prefixDirsPsr4;
$loader->prefixesPsr0 = ComposerStaticInitfd7e8d436e1dc450edc3153ac8bc31b4::$prefixesPsr0;
$loader->classMap = ComposerStaticInitfd7e8d436e1dc450edc3153ac8bc31b4::$classMap;
$loader->prefixLengthsPsr4 = ComposerStaticInitfc88bf644a3dc0d30b5792ad2fec7895::$prefixLengthsPsr4;
$loader->prefixDirsPsr4 = ComposerStaticInitfc88bf644a3dc0d30b5792ad2fec7895::$prefixDirsPsr4;
$loader->prefixesPsr0 = ComposerStaticInitfc88bf644a3dc0d30b5792ad2fec7895::$prefixesPsr0;
$loader->classMap = ComposerStaticInitfc88bf644a3dc0d30b5792ad2fec7895::$classMap;
}, null, ClassLoader::class);
}

276
vendor/composer/installed.json vendored
View File

@ -37,6 +37,89 @@
"description": "Simple validator library",
"homepage": "https://github.com/fguillot/simpleValidator"
},
{
"name": "fguillot/json-rpc",
"version": "v1.2.3",
"version_normalized": "1.2.3.0",
"source": {
"type": "git",
"url": "https://github.com/fguillot/JsonRPC.git",
"reference": "86dd3cf0724bc495cb31882122e9a3a77b9a2ffc"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/fguillot/JsonRPC/zipball/86dd3cf0724bc495cb31882122e9a3a77b9a2ffc",
"reference": "86dd3cf0724bc495cb31882122e9a3a77b9a2ffc",
"shasum": ""
},
"require": {
"php": ">=5.3.4"
},
"require-dev": {
"phpunit/phpunit": "4.8.*"
},
"time": "2016-12-15 03:00:30",
"type": "library",
"installation-source": "dist",
"autoload": {
"psr-0": {
"JsonRPC": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Frédéric Guillot"
}
],
"description": "Simple Json-RPC client/server library that just works",
"homepage": "https://github.com/fguillot/JsonRPC"
},
{
"name": "fguillot/picodb",
"version": "v1.0.14",
"version_normalized": "1.0.14.0",
"source": {
"type": "git",
"url": "https://github.com/fguillot/picoDb.git",
"reference": "86a831302ab10af800c83dbe4b3b01c88d5433f1"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/fguillot/picoDb/zipball/86a831302ab10af800c83dbe4b3b01c88d5433f1",
"reference": "86a831302ab10af800c83dbe4b3b01c88d5433f1",
"shasum": ""
},
"require": {
"php": ">=5.3.0"
},
"require-dev": {
"phpunit/phpunit": "4.8.*"
},
"time": "2016-07-16 22:59:59",
"type": "library",
"installation-source": "dist",
"autoload": {
"psr-0": {
"PicoDb": "lib/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Frédéric Guillot",
"homepage": "https://github.com/fguillot/"
}
],
"description": "Minimalist database query builder",
"homepage": "https://github.com/fguillot/picoDb"
},
{
"name": "zendframework/zendxml",
"version": "1.0.2",
@ -84,6 +167,61 @@
"zf2"
]
},
{
"name": "fguillot/picofeed",
"version": "v0.1.28",
"version_normalized": "0.1.28.0",
"source": {
"type": "git",
"url": "https://github.com/fguillot/picoFeed.git",
"reference": "9da506c308bcb40b6fc630f9123466028c03170b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/fguillot/picoFeed/zipball/9da506c308bcb40b6fc630f9123466028c03170b",
"reference": "9da506c308bcb40b6fc630f9123466028c03170b",
"shasum": ""
},
"require": {
"ext-dom": "*",
"ext-iconv": "*",
"ext-libxml": "*",
"ext-simplexml": "*",
"ext-xml": "*",
"php": ">=5.3.0",
"zendframework/zendxml": "^1.0"
},
"require-dev": {
"phpdocumentor/reflection-docblock": "2.0.4",
"phpunit/phpunit": "4.8.26",
"symfony/yaml": "2.8.7"
},
"suggest": {
"ext-curl": "PicoFeed will use cURL if present"
},
"time": "2016-12-29 00:06:41",
"bin": [
"picofeed"
],
"type": "library",
"installation-source": "dist",
"autoload": {
"psr-0": {
"PicoFeed": "lib/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Frédéric Guillot"
}
],
"description": "Modern library to handle RSS/Atom feeds",
"homepage": "https://github.com/fguillot/picoFeed"
},
{
"name": "pda/pheanstalk",
"version": "v3.1.0",
@ -136,48 +274,6 @@
"beanstalkd"
]
},
{
"name": "fguillot/picodb",
"version": "v1.0.14",
"version_normalized": "1.0.14.0",
"source": {
"type": "git",
"url": "https://github.com/fguillot/picoDb.git",
"reference": "86a831302ab10af800c83dbe4b3b01c88d5433f1"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/fguillot/picoDb/zipball/86a831302ab10af800c83dbe4b3b01c88d5433f1",
"reference": "86a831302ab10af800c83dbe4b3b01c88d5433f1",
"shasum": ""
},
"require": {
"php": ">=5.3.0"
},
"require-dev": {
"phpunit/phpunit": "4.8.*"
},
"time": "2016-07-16 22:59:59",
"type": "library",
"installation-source": "dist",
"autoload": {
"psr-0": {
"PicoDb": "lib/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Frédéric Guillot",
"homepage": "https://github.com/fguillot/"
}
],
"description": "Minimalist database query builder",
"homepage": "https://github.com/fguillot/picoDb"
},
{
"name": "ircmaxell/password-compat",
"version": "v1.0.4",
@ -221,101 +317,5 @@
"hashing",
"password"
]
},
{
"name": "fguillot/json-rpc",
"version": "v1.2.3",
"version_normalized": "1.2.3.0",
"source": {
"type": "git",
"url": "https://github.com/fguillot/JsonRPC.git",
"reference": "86dd3cf0724bc495cb31882122e9a3a77b9a2ffc"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/fguillot/JsonRPC/zipball/86dd3cf0724bc495cb31882122e9a3a77b9a2ffc",
"reference": "86dd3cf0724bc495cb31882122e9a3a77b9a2ffc",
"shasum": ""
},
"require": {
"php": ">=5.3.4"
},
"require-dev": {
"phpunit/phpunit": "4.8.*"
},
"time": "2016-12-15 03:00:30",
"type": "library",
"installation-source": "dist",
"autoload": {
"psr-0": {
"JsonRPC": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Frédéric Guillot"
}
],
"description": "Simple Json-RPC client/server library that just works",
"homepage": "https://github.com/fguillot/JsonRPC"
},
{
"name": "fguillot/picofeed",
"version": "v0.1.27",
"version_normalized": "0.1.27.0",
"source": {
"type": "git",
"url": "https://github.com/fguillot/picoFeed.git",
"reference": "41924841d3cd0480364ca9bcb90abe095d744457"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/fguillot/picoFeed/zipball/41924841d3cd0480364ca9bcb90abe095d744457",
"reference": "41924841d3cd0480364ca9bcb90abe095d744457",
"shasum": ""
},
"require": {
"ext-dom": "*",
"ext-iconv": "*",
"ext-libxml": "*",
"ext-simplexml": "*",
"ext-xml": "*",
"php": ">=5.3.0",
"zendframework/zendxml": "^1.0"
},
"require-dev": {
"phpdocumentor/reflection-docblock": "2.0.4",
"phpunit/phpunit": "4.8.26",
"symfony/yaml": "2.8.7"
},
"suggest": {
"ext-curl": "PicoFeed will use cURL if present"
},
"time": "2016-12-26 22:25:33",
"bin": [
"picofeed"
],
"type": "library",
"installation-source": "dist",
"autoload": {
"psr-0": {
"PicoFeed": "lib/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Frédéric Guillot"
}
],
"description": "Modern library to handle RSS/Atom feeds",
"homepage": "https://github.com/fguillot/picoFeed"
}
]

44
vendor/fguillot/picofeed/lib/PicoFeed/Client/Client.php vendored
View File

@ -2,6 +2,7 @@
namespace PicoFeed\Client;
use DateTime;
use LogicException;
use PicoFeed\Logging\Logger;
use PicoFeed\Config\Config;
@ -55,6 +56,13 @@ abstract class Client
*/
protected $last_modified = '';
/**
* Expiration DateTime
*
* @var DateTime
*/
protected $expiration = null;
/**
* Proxy hostname.
*
@ -214,6 +222,9 @@ abstract class Client
$this->handleErrorResponse($response);
$this->handleNormalResponse($response);
$this->expiration = $this->parseExpiration($response['headers']);
Logger::setMessage(get_called_class().' Expiration: '.$this->expiration->format(DATE_ISO8601));
return $this;
}
@ -241,6 +252,9 @@ abstract class Client
* Handle Http Error codes
*
* @param array $response Client response
* @throws ForbiddenException
* @throws InvalidUrlException
* @throws UnauthorizedException
*/
protected function handleErrorResponse(array $response)
{
@ -402,13 +416,12 @@ abstract class Client
/**
* Set the url.
*
* @param $url
* @return string
* @return \PicoFeed\Client\Client
*/
public function setUrl($url)
{
$this->url = $url;
return $this;
}
@ -670,4 +683,31 @@ abstract class Client
{
return $code == 301 || $code == 302 || $code == 303 || $code == 307;
}
public function parseExpiration(HttpHeaders $headers)
{
if (isset($headers['Cache-Control'])) {
if (preg_match('/s-maxage=(\d+)/', $headers['Cache-Control'], $matches)) {
return new DateTime('+' . $matches[1] . ' seconds');
} else if (preg_match('/max-age=(\d+)/', $headers['Cache-Control'], $matches)) {
return new DateTime('+' . $matches[1] . ' seconds');
}
}
if (! empty($headers['Expires'])) {
return new DateTime($headers['Expires']);
}
return new DateTime();
}
/**
* Get expiration date time from "Expires" or "Cache-Control" headers
*
* @return DateTime
*/
public function getExpiration()
{
return $this->expiration ?: new DateTime();
}
}