miniflux-legacy/vendor/fguillot/picofeed/lib/PicoFeed/Parser/Parser.php

577 lines
15 KiB
PHP
Raw Normal View History

2013-02-18 03:48:21 +01:00
<?php
namespace PicoFeed\Parser;
2013-02-18 03:48:21 +01:00
use SimpleXMLElement;
2015-04-28 18:08:42 +02:00
use PicoFeed\Client\Url;
use PicoFeed\Encoding\Encoding;
use PicoFeed\Filter\Filter;
use PicoFeed\Logging\Logger;
2015-04-28 18:08:42 +02:00
use PicoFeed\Scraper\Scraper;
2013-02-18 03:48:21 +01:00
2014-03-17 02:35:57 +01:00
/**
* Base parser class.
2014-03-17 02:35:57 +01:00
*
* @author Frederic Guillot
*/
2013-02-18 03:48:21 +01:00
abstract class Parser
{
2014-05-20 20:20:27 +02:00
/**
* Config object.
2014-05-20 20:20:27 +02:00
*
* @var \PicoFeed\Config\Config
2014-05-20 20:20:27 +02:00
*/
private $config;
2014-05-20 20:20:27 +02:00
2014-03-17 02:35:57 +01:00
/**
* DateParser object.
2014-03-17 02:35:57 +01:00
*
* @var \PicoFeed\Parser\DateParser
2014-03-17 02:35:57 +01:00
*/
protected $date;
2014-05-20 20:20:27 +02:00
/**
* Hash algorithm used to generate item id, any value supported by PHP, see hash_algos().
2014-05-20 20:20:27 +02:00
*
* @var string
*/
private $hash_algo = 'sha256';
2014-03-17 02:35:57 +01:00
/**
* Feed content (XML data).
2014-03-17 02:35:57 +01:00
*
* @var string
*/
2013-02-18 03:48:21 +01:00
protected $content = '';
/**
* Fallback url.
*
* @var string
*/
protected $fallback_url = '';
2014-03-17 02:35:57 +01:00
/**
* XML namespaces supported by parser.
2014-03-17 02:35:57 +01:00
*
2014-05-20 20:20:27 +02:00
* @var array
2014-03-17 02:35:57 +01:00
*/
2014-05-20 20:20:27 +02:00
protected $namespaces = array();
2014-03-17 02:35:57 +01:00
2015-07-19 17:19:26 +02:00
/**
* XML namespaces used in document.
2015-07-19 17:19:26 +02:00
*
* @var array
*/
protected $used_namespaces = array();
2014-10-19 20:42:31 +02:00
/**
* Enable the content filtering.
2014-10-19 20:42:31 +02:00
*
* @var bool
*/
private $enable_filter = true;
2014-03-17 02:35:57 +01:00
/**
* Enable the content grabber.
2014-03-17 02:35:57 +01:00
*
2014-05-20 20:20:27 +02:00
* @var bool
2014-03-17 02:35:57 +01:00
*/
2014-10-19 20:42:31 +02:00
private $enable_grabber = false;
2013-02-18 03:48:21 +01:00
2015-04-28 18:08:42 +02:00
/**
* Enable the content grabber on all pages.
2015-04-28 18:08:42 +02:00
*
* @var bool
*/
private $grabber_needs_rule_file = false;
2014-03-17 02:35:57 +01:00
/**
* Ignore those urls for the content scraper.
2014-03-17 02:35:57 +01:00
*
2014-05-20 20:20:27 +02:00
* @var array
2014-03-17 02:35:57 +01:00
*/
2014-05-20 20:20:27 +02:00
private $grabber_ignore_urls = array();
2013-02-18 03:48:21 +01:00
2014-03-17 02:35:57 +01:00
/**
* Constructor.
2014-03-17 02:35:57 +01:00
*
* @param string $content Feed content
* @param string $http_encoding HTTP encoding (headers)
* @param string $fallback_url Fallback url when the feed provide relative or broken url
2014-03-17 02:35:57 +01:00
*/
public function __construct($content, $http_encoding = '', $fallback_url = '')
2013-02-18 03:48:21 +01:00
{
$this->date = new DateParser();
$this->fallback_url = $fallback_url;
2014-05-25 14:47:03 +02:00
$xml_encoding = XmlParser::getEncodingFromXmlTag($content);
2013-10-04 05:14:39 +02:00
// Strip XML tag to avoid multiple encoding/decoding in the next XML processing
$this->content = Filter::stripXmlTag($content);
2013-06-01 20:57:17 +02:00
// Encode everything in UTF-8
Logger::setMessage(get_called_class().': HTTP Encoding "'.$http_encoding.'" ; XML Encoding "'.$xml_encoding.'"');
2014-05-25 14:47:03 +02:00
$this->content = Encoding::convert($this->content, $xml_encoding ?: $http_encoding);
2013-02-18 03:48:21 +01:00
}
2014-05-20 20:20:27 +02:00
/**
* Parse the document.
2014-05-20 20:20:27 +02:00
*
* @return \PicoFeed\Parser\Feed
2014-05-20 20:20:27 +02:00
*/
public function execute()
{
Logger::setMessage(get_called_class().': begin parsing');
2014-05-20 20:20:27 +02:00
$xml = XmlParser::getSimpleXml($this->content);
if ($xml === false) {
2015-06-21 15:56:36 +02:00
Logger::setMessage(get_called_class().': Applying XML workarounds');
$this->content = Filter::normalizeData($this->content);
$xml = XmlParser::getSimpleXml($this->content);
if ($xml === false) {
Logger::setMessage(get_called_class().': XML parsing error');
Logger::setMessage(XmlParser::getErrors());
throw new MalformedXmlException('XML parsing error');
}
2014-05-20 20:20:27 +02:00
}
2015-07-19 17:19:26 +02:00
$this->used_namespaces = $xml->getNamespaces(true);
$xml = $this->registerSupportedNamespaces($xml);
2014-05-20 20:20:27 +02:00
$feed = new Feed();
2014-05-20 20:20:27 +02:00
$this->findFeedUrl($xml, $feed);
$this->checkFeedUrl($feed);
$this->findSiteUrl($xml, $feed);
$this->checkSiteUrl($feed);
2014-05-20 20:20:27 +02:00
$this->findFeedTitle($xml, $feed);
2014-10-19 20:42:31 +02:00
$this->findFeedDescription($xml, $feed);
2014-05-20 20:20:27 +02:00
$this->findFeedLanguage($xml, $feed);
$this->findFeedId($xml, $feed);
$this->findFeedDate($xml, $feed);
2014-10-19 20:42:31 +02:00
$this->findFeedLogo($xml, $feed);
$this->findFeedIcon($xml, $feed);
2014-05-20 20:20:27 +02:00
foreach ($this->getItemsTree($xml) as $entry) {
2015-07-19 17:19:26 +02:00
$entry = $this->registerSupportedNamespaces($entry);
$item = new Item();
2015-03-26 00:59:41 +01:00
$item->xml = $entry;
2015-07-19 17:19:26 +02:00
$item->namespaces = $this->used_namespaces;
2015-03-26 00:59:41 +01:00
2014-05-20 20:20:27 +02:00
$this->findItemAuthor($xml, $entry, $item);
2014-05-20 20:20:27 +02:00
$this->findItemUrl($entry, $item);
$this->checkItemUrl($feed, $item);
2014-05-20 20:20:27 +02:00
$this->findItemTitle($entry, $item);
$this->findItemContent($entry, $item);
// Id generation can use the item url/title/content (order is important)
2014-05-20 20:20:27 +02:00
$this->findItemId($entry, $item, $feed);
$this->findItemDate($entry, $item, $feed);
2014-05-20 20:20:27 +02:00
$this->findItemEnclosure($entry, $item, $feed);
$this->findItemLanguage($entry, $item, $feed);
2014-10-19 20:42:31 +02:00
// Order is important (avoid double filtering)
2014-10-19 20:42:31 +02:00
$this->filterItemContent($feed, $item);
$this->scrapWebsite($item);
2014-10-19 20:42:31 +02:00
2014-05-20 20:20:27 +02:00
$feed->items[] = $item;
}
Logger::setMessage(get_called_class().PHP_EOL.$feed);
2014-05-20 20:20:27 +02:00
return $feed;
}
/**
* Check if the feed url is correct.
*
* @param Feed $feed Feed object
*/
public function checkFeedUrl(Feed $feed)
{
if ($feed->getFeedUrl() === '') {
$feed->feed_url = $this->fallback_url;
} else {
$feed->feed_url = Url::resolve($feed->getFeedUrl(), $this->fallback_url);
}
}
/**
* Check if the site url is correct.
*
* @param Feed $feed Feed object
*/
public function checkSiteUrl(Feed $feed)
{
if ($feed->getSiteUrl() === '') {
$feed->site_url = Url::base($feed->getFeedUrl());
} else {
$feed->site_url = Url::resolve($feed->getSiteUrl(), $this->fallback_url);
}
}
/**
* Check if the item url is correct.
*
* @param Feed $feed Feed object
* @param Item $item Item object
*/
public function checkItemUrl(Feed $feed, Item $item)
{
$item->url = Url::resolve($item->getUrl(), $feed->getSiteUrl());
}
2014-03-17 02:35:57 +01:00
/**
* Fetch item content with the content grabber.
2014-03-17 02:35:57 +01:00
*
* @param Item $item Item object
2014-03-17 02:35:57 +01:00
*/
2014-10-19 20:42:31 +02:00
public function scrapWebsite(Item $item)
2013-02-18 03:48:21 +01:00
{
if ($this->enable_grabber && !in_array($item->getUrl(), $this->grabber_ignore_urls)) {
2015-04-28 18:08:42 +02:00
$grabber = new Scraper($this->config);
$grabber->setUrl($item->getUrl());
if ($this->grabber_needs_rule_file) {
$grabber->disableCandidateParser();
}
$grabber->execute();
2014-05-20 20:20:27 +02:00
2015-04-28 18:08:42 +02:00
if ($grabber->hasRelevantContent()) {
$item->content = $grabber->getFilteredContent();
2014-05-20 20:20:27 +02:00
}
}
2013-02-18 03:48:21 +01:00
}
2014-03-17 02:35:57 +01:00
/**
* Filter HTML for entry content.
2014-03-17 02:35:57 +01:00
*
* @param Feed $feed Feed object
* @param Item $item Item object
2014-03-17 02:35:57 +01:00
*/
2014-10-19 20:42:31 +02:00
public function filterItemContent(Feed $feed, Item $item)
2013-02-18 03:48:21 +01:00
{
2014-10-19 20:42:31 +02:00
if ($this->isFilteringEnabled()) {
$filter = Filter::html($item->getContent(), $feed->getSiteUrl());
2014-10-19 20:42:31 +02:00
$filter->setConfig($this->config);
$item->content = $filter->execute();
} else {
Logger::setMessage(get_called_class().': Content filtering disabled');
}
}
2014-03-17 02:35:57 +01:00
/**
* Generate a unique id for an entry (hash all arguments).
2014-03-17 02:35:57 +01:00
*
* @return string
2014-03-17 02:35:57 +01:00
*/
public function generateId()
{
2014-05-20 20:20:27 +02:00
return hash($this->hash_algo, implode(func_get_args()));
}
2013-10-24 00:39:21 +02:00
2014-03-17 02:35:57 +01:00
/**
* Return true if the given language is "Right to Left".
2014-03-17 02:35:57 +01:00
*
* @static
*
* @param string $language Language: fr-FR, en-US
*
2014-03-17 02:35:57 +01:00
* @return bool
*/
public static function isLanguageRTL($language)
{
$language = strtolower($language);
2014-05-20 20:20:27 +02:00
$rtl_languages = array(
'ar', // Arabic (ar-**)
'fa', // Farsi (fa-**)
'ur', // Urdu (ur-**)
'ps', // Pashtu (ps-**)
'syr', // Syriac (syr-**)
'dv', // Divehi (dv-**)
'he', // Hebrew (he-**)
'yi', // Yiddish (yi-**)
);
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
foreach ($rtl_languages as $prefix) {
if (strpos($language, $prefix) === 0) {
return true;
}
}
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
return false;
}
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
/**
* Set Hash algorithm used for id generation.
*
* @param string $algo Algorithm name
2014-05-20 20:20:27 +02:00
*
* @return \PicoFeed\Parser\Parser
2014-05-20 20:20:27 +02:00
*/
public function setHashAlgo($algo)
{
$this->hash_algo = $algo ?: $this->hash_algo;
2014-05-20 20:20:27 +02:00
return $this;
}
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
/**
* Set a different timezone.
2014-05-20 20:20:27 +02:00
*
* @see http://php.net/manual/en/timezones.php
*
* @param string $timezone Timezone
*
* @return \PicoFeed\Parser\Parser
2014-05-20 20:20:27 +02:00
*/
public function setTimezone($timezone)
{
if ($timezone) {
$this->date->timezone = $timezone;
}
2014-05-20 20:20:27 +02:00
return $this;
}
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
/**
* Set config object.
*
* @param \PicoFeed\Config\Config $config Config instance
2014-05-20 20:20:27 +02:00
*
* @return \PicoFeed\Parser\Parser
2014-05-20 20:20:27 +02:00
*/
public function setConfig($config)
{
$this->config = $config;
2014-05-20 20:20:27 +02:00
return $this;
}
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
/**
* Enable the content grabber.
2014-05-20 20:20:27 +02:00
*
* @return \PicoFeed\Parser\Parser
2014-05-20 20:20:27 +02:00
*/
2014-10-19 20:42:31 +02:00
public function disableContentFiltering()
2014-05-20 20:20:27 +02:00
{
2014-10-19 20:42:31 +02:00
$this->enable_filter = false;
2014-05-20 20:20:27 +02:00
}
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
/**
* Return true if the content filtering is enabled.
2014-05-20 20:20:27 +02:00
*
* @return bool
2014-05-20 20:20:27 +02:00
*/
2014-10-19 20:42:31 +02:00
public function isFilteringEnabled()
2014-05-20 20:20:27 +02:00
{
2014-10-19 20:42:31 +02:00
if ($this->config === null) {
return $this->enable_filter;
}
return $this->config->getContentFiltering($this->enable_filter);
2014-05-20 20:20:27 +02:00
}
2014-03-17 02:35:57 +01:00
2014-05-20 20:20:27 +02:00
/**
* Enable the content grabber.
2014-05-20 20:20:27 +02:00
*
2015-04-28 18:08:42 +02:00
* @param bool $needs_rule_file true if only pages with rule files should be
* scraped
*
* @return \PicoFeed\Parser\Parser
2014-05-20 20:20:27 +02:00
*/
2015-04-28 18:08:42 +02:00
public function enableContentGrabber($needs_rule_file = false)
2014-05-20 20:20:27 +02:00
{
2014-10-19 20:42:31 +02:00
$this->enable_grabber = true;
2015-04-28 18:08:42 +02:00
$this->grabber_needs_rule_file = $needs_rule_file;
2014-10-19 20:42:31 +02:00
}
2014-05-20 20:20:27 +02:00
2014-10-19 20:42:31 +02:00
/**
* Set ignored URLs for the content grabber.
*
* @param array $urls URLs
2014-10-19 20:42:31 +02:00
*
* @return \PicoFeed\Parser\Parser
2014-10-19 20:42:31 +02:00
*/
public function setGrabberIgnoreUrls(array $urls)
{
$this->grabber_ignore_urls = $urls;
2014-03-17 02:35:57 +01:00
}
2015-07-19 17:19:26 +02:00
/**
* Register all supported namespaces to be used within an xpath query.
*
* @param SimpleXMLElement $xml Feed xml
2015-07-19 17:19:26 +02:00
*
* @return SimpleXMLElement
*/
public function registerSupportedNamespaces(SimpleXMLElement $xml)
{
foreach ($this->namespaces as $prefix => $ns) {
$xml->registerXPathNamespace($prefix, $ns);
}
return $xml;
}
/**
* Find the feed url.
*
* @param SimpleXMLElement $xml Feed xml
* @param \PicoFeed\Parser\Feed $feed Feed object
*/
abstract public function findFeedUrl(SimpleXMLElement $xml, Feed $feed);
/**
* Find the site url.
*
* @param SimpleXMLElement $xml Feed xml
* @param \PicoFeed\Parser\Feed $feed Feed object
*/
abstract public function findSiteUrl(SimpleXMLElement $xml, Feed $feed);
/**
* Find the feed title.
*
* @param SimpleXMLElement $xml Feed xml
* @param \PicoFeed\Parser\Feed $feed Feed object
*/
abstract public function findFeedTitle(SimpleXMLElement $xml, Feed $feed);
/**
* Find the feed description.
*
* @param SimpleXMLElement $xml Feed xml
* @param \PicoFeed\Parser\Feed $feed Feed object
*/
abstract public function findFeedDescription(SimpleXMLElement $xml, Feed $feed);
/**
* Find the feed language.
*
* @param SimpleXMLElement $xml Feed xml
* @param \PicoFeed\Parser\Feed $feed Feed object
*/
abstract public function findFeedLanguage(SimpleXMLElement $xml, Feed $feed);
/**
* Find the feed id.
*
* @param SimpleXMLElement $xml Feed xml
* @param \PicoFeed\Parser\Feed $feed Feed object
*/
abstract public function findFeedId(SimpleXMLElement $xml, Feed $feed);
/**
* Find the feed date.
*
* @param SimpleXMLElement $xml Feed xml
* @param \PicoFeed\Parser\Feed $feed Feed object
*/
abstract public function findFeedDate(SimpleXMLElement $xml, Feed $feed);
/**
* Find the feed logo url.
*
* @param SimpleXMLElement $xml Feed xml
* @param \PicoFeed\Parser\Feed $feed Feed object
*/
abstract public function findFeedLogo(SimpleXMLElement $xml, Feed $feed);
/**
* Find the feed icon.
*
* @param SimpleXMLElement $xml Feed xml
* @param \PicoFeed\Parser\Feed $feed Feed object
*/
abstract public function findFeedIcon(SimpleXMLElement $xml, Feed $feed);
/**
* Get the path to the items XML tree.
*
* @param SimpleXMLElement $xml Feed xml
*
* @return SimpleXMLElement
*/
abstract public function getItemsTree(SimpleXMLElement $xml);
/**
* Find the item author.
*
* @param SimpleXMLElement $xml Feed
* @param SimpleXMLElement $entry Feed item
* @param \PicoFeed\Parser\Item $item Item object
*/
abstract public function findItemAuthor(SimpleXMLElement $xml, SimpleXMLElement $entry, Item $item);
/**
* Find the item URL.
*
* @param SimpleXMLElement $entry Feed item
* @param \PicoFeed\Parser\Item $item Item object
*/
abstract public function findItemUrl(SimpleXMLElement $entry, Item $item);
/**
* Find the item title.
*
* @param SimpleXMLElement $entry Feed item
* @param \PicoFeed\Parser\Item $item Item object
*/
abstract public function findItemTitle(SimpleXMLElement $entry, Item $item);
/**
* Genereate the item id.
*
* @param SimpleXMLElement $entry Feed item
* @param \PicoFeed\Parser\Item $item Item object
* @param \PicoFeed\Parser\Feed $feed Feed object
*/
abstract public function findItemId(SimpleXMLElement $entry, Item $item, Feed $feed);
/**
* Find the item date.
*
* @param SimpleXMLElement $entry Feed item
* @param Item $item Item object
* @param \PicoFeed\Parser\Feed $feed Feed object
*/
abstract public function findItemDate(SimpleXMLElement $entry, Item $item, Feed $feed);
/**
* Find the item content.
*
* @param SimpleXMLElement $entry Feed item
* @param \PicoFeed\Parser\Item $item Item object
*/
abstract public function findItemContent(SimpleXMLElement $entry, Item $item);
/**
* Find the item enclosure.
*
* @param SimpleXMLElement $entry Feed item
* @param \PicoFeed\Parser\Item $item Item object
* @param \PicoFeed\Parser\Feed $feed Feed object
*/
abstract public function findItemEnclosure(SimpleXMLElement $entry, Item $item, Feed $feed);
/**
* Find the item language.
*
* @param SimpleXMLElement $entry Feed item
* @param \PicoFeed\Parser\Item $item Item object
* @param \PicoFeed\Parser\Feed $feed Feed object
*/
abstract public function findItemLanguage(SimpleXMLElement $entry, Item $item, Feed $feed);
2013-02-18 03:48:21 +01:00
}