miniflux-legacy/vendor/PicoFeed/Parser.php

471 lines
12 KiB
PHP
Raw Normal View History

2013-02-18 03:48:21 +01:00
<?php
namespace PicoFeed;
2014-05-20 20:20:27 +02:00
use DateTime;
use DateTimeZone;
2013-02-18 03:48:21 +01:00
2014-03-17 02:35:57 +01:00
/**
* Base parser class
*
* @author Frederic Guillot
* @package parser
*/
2013-02-18 03:48:21 +01:00
abstract class Parser
{
2014-05-20 20:20:27 +02:00
/**
* Config object
*
* @access private
* @var \PicoFeed\Config
*/
private $config = null;
2014-03-17 02:35:57 +01:00
/**
* Hash algorithm used to generate item id, any value supported by PHP, see hash_algos()
*
2014-05-20 20:20:27 +02:00
* @access private
2014-03-17 02:35:57 +01:00
* @var string
*/
2014-05-20 20:20:27 +02:00
private $hash_algo = 'crc32b'; // crc32b seems to be faster and shorter than other hash algorithms
/**
* Timezone used to parse feed dates
*
* @access private
* @var string
*/
private $timezone = 'UTC';
2014-03-17 02:35:57 +01:00
/**
* Feed content (XML data)
*
* @access protected
* @var string
*/
2013-02-18 03:48:21 +01:00
protected $content = '';
/**
* Fallback url
*
* @access protected
* @var string
*/
protected $fallback_url = '';
2014-03-17 02:35:57 +01:00
/**
2014-05-20 20:20:27 +02:00
* XML namespaces
2014-03-17 02:35:57 +01:00
*
2014-05-20 20:20:27 +02:00
* @access protected
* @var array
2014-03-17 02:35:57 +01:00
*/
2014-05-20 20:20:27 +02:00
protected $namespaces = array();
2014-03-17 02:35:57 +01:00
2014-10-19 20:42:31 +02:00
/**
* Enable the content filtering
*
* @access private
* @var bool
*/
private $enable_filter = true;
2014-03-17 02:35:57 +01:00
/**
2014-05-20 20:20:27 +02:00
* Enable the content grabber
2014-03-17 02:35:57 +01:00
*
2014-05-20 20:20:27 +02:00
* @access private
* @var bool
2014-03-17 02:35:57 +01:00
*/
2014-10-19 20:42:31 +02:00
private $enable_grabber = false;
2013-02-18 03:48:21 +01:00
2014-03-17 02:35:57 +01:00
/**
2014-05-20 20:20:27 +02:00
* Ignore those urls for the content scraper
2014-03-17 02:35:57 +01:00
*
2014-05-20 20:20:27 +02:00
* @access private
* @var array
2014-03-17 02:35:57 +01:00
*/
2014-05-20 20:20:27 +02:00
private $grabber_ignore_urls = array();
2013-02-18 03:48:21 +01:00
2014-03-17 02:35:57 +01:00
/**
* Constructor
*
* @access public
* @param string $content Feed content
* @param string $http_encoding HTTP encoding (headers)
* @param string $base_url Fallback url when the feed provide relative or broken url
2014-03-17 02:35:57 +01:00
*/
public function __construct($content, $http_encoding = '', $fallback_url = '')
2013-02-18 03:48:21 +01:00
{
$this->fallback_url = $fallback_url;
2014-05-25 14:47:03 +02:00
$xml_encoding = XmlParser::getEncodingFromXmlTag($content);
2013-10-04 05:14:39 +02:00
// Strip XML tag to avoid multiple encoding/decoding in the next XML processing
$this->content = Filter::stripXmlTag($content);
2013-06-01 20:57:17 +02:00
// Encode everything in UTF-8
2014-05-25 14:47:03 +02:00
Logging::setMessage(get_called_class().': HTTP Encoding "'.$http_encoding.'" ; XML Encoding "'.$xml_encoding.'"');
$this->content = Encoding::convert($this->content, $xml_encoding ?: $http_encoding);
2013-06-01 20:57:17 +02:00
// Workarounds
2014-10-19 20:42:31 +02:00
$this->content = Filter::normalizeData($this->content);
2013-02-18 03:48:21 +01:00
}
2014-05-20 20:20:27 +02:00
/**
* Parse the document
*
* @access public
* @return mixed \PicoFeed\Feed instance or false
*/
public function execute()
{
Logging::setMessage(get_called_class().': begin parsing');
$xml = XmlParser::getSimpleXml($this->content);
if ($xml === false) {
Logging::setMessage(get_called_class().': XML parsing error');
Logging::setMessage(XmlParser::getErrors());
return false;
}
$this->namespaces = $xml->getNamespaces(true);
$feed = new Feed;
2014-05-20 20:20:27 +02:00
$this->findFeedUrl($xml, $feed);
$this->checkFeedUrl($feed);
2014-05-20 20:20:27 +02:00
$this->findFeedTitle($xml, $feed);
2014-10-19 20:42:31 +02:00
$this->findFeedDescription($xml, $feed);
2014-05-20 20:20:27 +02:00
$this->findFeedLanguage($xml, $feed);
$this->findFeedId($xml, $feed);
$this->findFeedDate($xml, $feed);
2014-10-19 20:42:31 +02:00
$this->findFeedLogo($xml, $feed);
2014-05-20 20:20:27 +02:00
foreach ($this->getItemsTree($xml) as $entry) {
$item = new Item;
$this->findItemAuthor($xml, $entry, $item);
2014-05-20 20:20:27 +02:00
$this->findItemUrl($entry, $item);
$this->checkItemUrl($feed, $item);
2014-05-20 20:20:27 +02:00
$this->findItemTitle($entry, $item);
$this->findItemId($entry, $item, $feed);
$this->findItemDate($entry, $item);
$this->findItemContent($entry, $item);
$this->findItemEnclosure($entry, $item, $feed);
$this->findItemLanguage($entry, $item, $feed);
2014-10-19 20:42:31 +02:00
$this->scrapWebsite($item);
$this->filterItemContent($feed, $item);
2014-05-20 20:20:27 +02:00
$feed->items[] = $item;
}
Logging::setMessage(get_called_class().PHP_EOL.$feed);
return $feed;
}
/**
* Check if the feed url is correct
*
* @access public
* @param Feed $feed Feed object
*/
public function checkFeedUrl(Feed $feed)
{
$url = new Url($feed->getUrl());
if ($url->isRelativeUrl()) {
$feed->url = $this->fallback_url;
}
}
/**
* Check if the item url is correct
*
* @access public
* @param Feed $feed Feed object
* @param Item $item Item object
*/
public function checkItemUrl(Feed $feed, Item $item)
{
$url = new Url($item->getUrl());
if ($url->isRelativeUrl()) {
$item->url = Url::resolve($item->getUrl(), $feed->getUrl());
}
}
2014-03-17 02:35:57 +01:00
/**
2014-10-19 20:42:31 +02:00
* Fetch item content with the content grabber
2014-03-17 02:35:57 +01:00
*
* @access public
2014-10-19 20:42:31 +02:00
* @param Item $item Item object
2014-03-17 02:35:57 +01:00
*/
2014-10-19 20:42:31 +02:00
public function scrapWebsite(Item $item)
2013-02-18 03:48:21 +01:00
{
2014-10-19 20:42:31 +02:00
if ($this->enable_grabber && ! in_array($item->getUrl(), $this->grabber_ignore_urls)) {
2014-05-20 20:20:27 +02:00
2014-10-19 20:42:31 +02:00
$grabber = new Grabber($item->getUrl());
2014-05-20 20:20:27 +02:00
$grabber->setConfig($this->config);
$grabber->download();
if ($grabber->parse()) {
2014-10-19 20:42:31 +02:00
$item->content = $grabber->getContent() ?: $item->content;
2014-05-20 20:20:27 +02:00
}
}
2013-02-18 03:48:21 +01:00
}
2014-03-17 02:35:57 +01:00
/**
2014-10-19 20:42:31 +02:00
* Filter HTML for entry content
2014-03-17 02:35:57 +01:00
*
* @access public
2014-10-19 20:42:31 +02:00
* @param Feed $feed Feed object
* @param Item $item Item object
2014-03-17 02:35:57 +01:00
*/
2014-10-19 20:42:31 +02:00
public function filterItemContent(Feed $feed, Item $item)
2013-02-18 03:48:21 +01:00
{
2014-10-19 20:42:31 +02:00
if ($this->isFilteringEnabled()) {
$filter = Filter::html($item->getContent(), $feed->getUrl());
$filter->setConfig($this->config);
$item->content = $filter->execute();
}
else {
Logging::setMessage(get_called_class().': Content filtering disabled');
}
}
2014-03-17 02:35:57 +01:00
/**
* Generate a unique id for an entry (hash all arguments)
*
* @access public
* @param string $args Pieces of data to hash
* @return string Id
*/
public function generateId()
{
2014-05-20 20:20:27 +02:00
return hash($this->hash_algo, implode(func_get_args()));
}
2013-10-24 00:39:21 +02:00
2014-03-17 02:35:57 +01:00
/**
* Try to parse all date format for broken feeds
*
* @access public
* @param string $value Original date format
* @return integer Timestamp
*/
2013-10-24 00:39:21 +02:00
public function parseDate($value)
{
// Format => truncate to this length if not null
$formats = array(
DATE_ATOM => null,
DATE_RSS => null,
DATE_COOKIE => null,
DATE_ISO8601 => null,
DATE_RFC822 => null,
DATE_RFC850 => null,
DATE_RFC1036 => null,
DATE_RFC1123 => null,
DATE_RFC2822 => null,
DATE_RFC3339 => null,
'D, d M Y H:i:s' => 25,
'D, d M Y h:i:s' => 25,
'D M d Y H:i:s' => 24,
'Y-m-d H:i:s' => 19,
'Y-m-d\TH:i:s' => 19,
'd/m/Y H:i:s' => 19,
'D, d M Y' => 16,
'Y-m-d' => 10,
'd-m-Y' => 10,
'm-d-Y' => 10,
'd.m.Y' => 10,
'm.d.Y' => 10,
'd/m/Y' => 10,
'm/d/Y' => 10,
);
$value = trim($value);
foreach ($formats as $format => $length) {
$truncated_value = $value;
if ($length !== null) {
$truncated_value = substr($truncated_value, 0, $length);
}
$timestamp = $this->getValidDate($format, $truncated_value);
if ($timestamp > 0) {
return $timestamp;
}
2013-10-24 00:39:21 +02:00
}
2014-05-20 20:20:27 +02:00
$date = new DateTime('now', new DateTimeZone($this->timezone));
return $date->getTimestamp();
2013-10-24 00:39:21 +02:00
}
2014-03-17 02:35:57 +01:00
/**
* Get a valid date from a given format
*
* @access public
* @param string $format Date format
* @param string $value Original date value
* @return integer Timestamp
*/
2013-10-24 00:39:21 +02:00
public function getValidDate($format, $value)
{
2014-05-20 20:20:27 +02:00
$date = DateTime::createFromFormat($format, $value, new DateTimeZone($this->timezone));
2013-10-24 00:39:21 +02:00
if ($date !== false) {
2014-05-20 20:20:27 +02:00
$errors = DateTime::getLastErrors();
if ($errors['error_count'] === 0 && $errors['warning_count'] === 0) {
return $date->getTimestamp();
}
2013-10-24 00:39:21 +02:00
}
return 0;
}
2013-12-18 02:56:53 +01:00
2014-03-17 02:35:57 +01:00
/**
* Hardcoded list of hostname/token to exclude from id generation
*
* @access public
* @param string $url URL
* @return boolean
*/
2013-12-18 02:56:53 +01:00
public function isExcludedFromId($url)
{
2014-02-01 20:52:33 +01:00
$exclude_list = array('ap.org', 'jacksonville.com');
2013-12-18 02:56:53 +01:00
foreach ($exclude_list as $token) {
if (strpos($url, $token) !== false) return true;
}
return false;
}
2014-03-17 02:35:57 +01:00
/**
* Return true if the given language is "Right to Left"
*
* @static
* @access public
* @param string $language Language: fr-FR, en-US
* @return bool
*/
public static function isLanguageRTL($language)
{
$language = strtolower($language);
2014-05-20 20:20:27 +02:00
$rtl_languages = array(
'ar', // Arabic (ar-**)
'fa', // Farsi (fa-**)
'ur', // Urdu (ur-**)
'ps', // Pashtu (ps-**)
'syr', // Syriac (syr-**)
'dv', // Divehi (dv-**)
'he', // Hebrew (he-**)
'yi', // Yiddish (yi-**)
);
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
foreach ($rtl_languages as $prefix) {
if (strpos($language, $prefix) === 0) {
return true;
}
}
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
return false;
}
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
/**
* Set Hash algorithm used for id generation
*
* @access public
* @param string $algo Algorithm name
* @return \PicoFeed\Parser
*/
public function setHashAlgo($algo)
{
$this->hash_algo = $algo ?: $this->hash_algo;
return $this;
}
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
/**
* Set a different timezone
*
* @see http://php.net/manual/en/timezones.php
* @access public
* @param string $timezone Timezone
* @return \PicoFeed\Parser
*/
public function setTimezone($timezone)
{
$this->timezone = $timezone ?: $this->timezone;
return $this;
}
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
/**
* Set config object
*
* @access public
* @param \PicoFeed\Config $config Config instance
* @return \PicoFeed\Parser
*/
public function setConfig($config)
{
$this->config = $config;
return $this;
}
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
/**
* Enable the content grabber
*
* @access public
* @return \PicoFeed\Parser
*/
2014-10-19 20:42:31 +02:00
public function disableContentFiltering()
2014-05-20 20:20:27 +02:00
{
2014-10-19 20:42:31 +02:00
$this->enable_filter = false;
2014-05-20 20:20:27 +02:00
}
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
/**
2014-10-19 20:42:31 +02:00
* Return true if the content filtering is enabled
2014-05-20 20:20:27 +02:00
*
* @access public
2014-10-19 20:42:31 +02:00
* @return boolean
2014-05-20 20:20:27 +02:00
*/
2014-10-19 20:42:31 +02:00
public function isFilteringEnabled()
2014-05-20 20:20:27 +02:00
{
2014-10-19 20:42:31 +02:00
if ($this->config === null) {
return $this->enable_filter;
}
return $this->config->getContentFiltering($this->enable_filter);
2014-05-20 20:20:27 +02:00
}
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
/**
2014-10-19 20:42:31 +02:00
* Enable the content grabber
2014-05-20 20:20:27 +02:00
*
* @access public
2014-10-19 20:42:31 +02:00
* @return \PicoFeed\Parser
2014-05-20 20:20:27 +02:00
*/
2014-10-19 20:42:31 +02:00
public function enableContentGrabber()
2014-05-20 20:20:27 +02:00
{
2014-10-19 20:42:31 +02:00
$this->enable_grabber = true;
}
2014-05-20 20:20:27 +02:00
2014-10-19 20:42:31 +02:00
/**
* Set ignored URLs for the content grabber
*
* @access public
* @param array $urls URLs
* @return \PicoFeed\Parser
*/
public function setGrabberIgnoreUrls(array $urls)
{
$this->grabber_ignore_urls = $urls;
2014-03-17 02:35:57 +01:00
}
2013-02-18 03:48:21 +01:00
}