From 2bb7c4b1402fcce0f4fab02b2ac8210ba5d8d613 Mon Sep 17 00:00:00 2001 From: Nikolai R Kristiansen Date: Sat, 18 Jun 2016 14:48:00 +0200 Subject: [PATCH] Oembed is nice, download more html --- README | 24 ---------------- README.md | 30 +++++++++++++++++++ requirements.txt | 5 ++-- snurr.py | 26 +++-------------- tests.py | 16 +++++------ utils.py | 75 ++++++++++++++++++++++++++++++++++++------------ 6 files changed, 100 insertions(+), 76 deletions(-) delete mode 100644 README create mode 100644 README.md diff --git a/README b/README deleted file mode 100644 index 5264e1a..0000000 --- a/README +++ /dev/null @@ -1,24 +0,0 @@ -# Requirements: -With virtualenv: - apt install libxml2-dev libxslt1-dev python-dev libmysqlclient-dev - virtualenv venv - . venv/bin/activate - pip install -r requirements.txt - -Without venv: - python-twisted python-mysqldb python-lxml python-openssl python-requests - -$ python snurr.py -h -Usage: python snurr.py [options] CHANNEL - -Pipes UDP-messages to an IRC-channel. - -Options: - -h, --help show this help message and exit - -c SERVER, --connect=SERVER - IRC server (default: irc.ifi.uio.no) - -p PORT, --port=PORT IRC server port (default: 6667) - -s, --ssl connect with SSL (default: False) - -l LISTEN_PORT, --listen_port=LISTEN_PORT - UDP listen port (default: 55666) - diff --git a/README.md b/README.md new file mode 100644 index 0000000..5075da1 --- /dev/null +++ b/README.md @@ -0,0 +1,30 @@ +Friendly IRC bot using Twisted (python 2.7) + +# Requirements: +With virtualenv: + + apt install libxml2-dev libxslt1-dev python-dev libmysqlclient-dev + virtualenv venv + . venv/bin/activate + pip install -r requirements.txt + +Without venv: + + apt install python-twisted python-mysqldb python-lxml python-openssl python-requests + +Usage: + + $ python snurr.py -h + Usage: python snurr.py [-h] [options] CHANNEL + + Pipes UDP-messages to an IRC-channel. + + Options: + -h, --help show this help message and exit + -c SERVER, --connect=SERVER + IRC server (default: irc.oftc.net) + -p PORT, --port=PORT IRC server port (default: 6697) + -s, --ssl connect with SSL (default: False) + -l LISTEN_PORT, --listen_port=LISTEN_PORT + UDP listen port (default: 55666) + diff --git a/requirements.txt b/requirements.txt index 05184ca..3b9f38d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -Twisted[tls]==16.0.0 +Twisted[tls]==16.2.0 lxml==3.6.0 +micawber==0.3.3 mysqlclient==1.3.7 -requests==2.9.1 +requests==2.10.0 ndg-httpsclient==0.4.0 \ No newline at end of file diff --git a/snurr.py b/snurr.py index b223cd4..b660422 100755 --- a/snurr.py +++ b/snurr.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # encoding: utf-8 import optparse -import requests import re import logging from twisted.words.protocols import irc @@ -9,7 +8,7 @@ from twisted.internet import protocol, reactor, ssl import settings -from utils import ReconnectingConnectionPool, ping_host, parse_title +from utils import ReconnectingConnectionPool, ping_host, get_reply_from_url logger = logging.getLogger('snurr') @@ -157,27 +156,10 @@ class IRCActions: # Check if msg contains HTTP(S) URL urlmatch = re.search(r"(https?):\/\/([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w\.-]*\/?)(\?[\/\w=&\.-]+)?", msg, re.I) if urlmatch is not None: - self.msg_urlinfo(urlmatch, channel, nick) - - def msg_urlinfo(self, urlmatch, channel, nick): - reply = "URL: " - url = urlmatch.group() - try: - req = requests.get(url, headers={'User-Agent': 'snurrbot v0.1'}, stream=True) - except Exception as e: - logger.info(e) - self.bot.msgReply(nick, channel, reply + "Timeout") - return - - content_type = req.headers['content-type'] - if ';' in content_type: - content_type = content_type.split(';')[0] - reply += "%d %s " % (req.status_code, content_type) - if req.status_code == 200 and content_type == "text/html": - # Read max 20480 bytes - partial_html = next(req.iter_content(chunk_size=20480)) - reply += parse_title(partial_html) + self.msg_urlinfo(urlmatch.group(), channel, nick) + def msg_urlinfo(self, url, channel, nick): + reply = get_reply_from_url(url) self.bot.msgReply(nick, channel, reply) def get_tetris_highscore(self): diff --git a/tests.py b/tests.py index 0ec0586..525a2a8 100644 --- a/tests.py +++ b/tests.py @@ -1,7 +1,7 @@ +from __future__ import unicode_literals import unittest -import requests -from utils import parse_title +from utils import parse_title, get_reply_from_url class ParseHTMLTest(unittest.TestCase): @@ -12,14 +12,12 @@ class ParseHTMLTest(unittest.TestCase): {} """.format(title) - self.assertEqual(parse_title(partial_html=partial_html), '[{}]'.format(title)) + self.assertEqual(parse_title(html=partial_html), title) - # @unittest.skip() - def test_parse_title_from_stream(self): - url = 'http://lol.com' - req = requests.get(url, headers={'User-Agent': 'snurrbot v0.1'}, stream=True) - partial_html = req.iter_content(chunk_size=10240) - self.assertEqual(parse_title(partial_html=partial_html), None) + def test_get_reply_from_url(self): + url = 'https://www.youtube.com/watch?v=3yR_BAqB7aQ' + reply = get_reply_from_url(url) + self.assertEqual(reply, 'URL: 200 text/html [Daydream Labs: Puzzle - YouTube]') if __name__ == '__main__': unittest.main() diff --git a/utils.py b/utils.py index 99cd18c..139cb61 100644 --- a/utils.py +++ b/utils.py @@ -1,8 +1,12 @@ -import logging -import re -import subprocess +from __future__ import unicode_literals +import logging import lxml.html +from lxml.etree import XMLSyntaxError +import micawber +import re +import requests +import subprocess from twisted.enterprise import adbapi @@ -53,22 +57,55 @@ def ping_host(host): return "feil med ping:" + str(e) -def parse_title(partial_html): - reply = '' +def get_title_with_oembed(url): + providers = micawber.bootstrap_basic() try: - # This might raise an exception if the HTML is _really_ broken - tree = lxml.html.fromstring(partial_html) - title = tree.find(".//title") - if title is not None and title.text is not None: - reply += "[%s]" % re.sub(r'\s+', ' ', title.text.strip()) - # DEBUG - if title is None: - reply += "" - elif title.text is None: - reply += "<title.text is none>" - # END - except Exception as e: - reply += "<exception>" - logger.error(e.message) + res = providers.request(url) + except micawber.ProviderException as e: + logger.info(e) + return '' + + return '{} - {}'.format(res.get('title'), res.get('provider_name')) + + +def get_reply_from_url(url): + user_agent = 'snurrbot v0.1' + + # Try using oembed + title = get_title_with_oembed(url) + + # Fetch the html + try: + req = requests.get(url, headers={'User-Agent': user_agent}) + except requests.Timeout as e: + logger.info(e) + return 'URL Timeout' + + content_type = req.headers['content-type'] + if ';' in content_type: + content_type = content_type.split(';')[0] + + if not title and req.status_code == 200 and content_type == "text/html": + title = parse_title(req.text) + + title = '' if not title else ' [{}]'.format(title) + reply = "URL: {} {}{}".format(req.status_code, content_type, title) return reply + + +def parse_title(html): + title = '' + + try: + # This might raise an exception if the HTML is _really_ broken + tree = lxml.html.fromstring(html) + title_tag = tree.find(".//title") + + if title_tag is not None and title_tag.text is not None: + title += "%s" % re.sub(r'\s+', ' ', title_tag.text.strip()) + + except XMLSyntaxError as e: + logger.info(str(e)) + + return title