diff --git a/README b/README
deleted file mode 100644
index 5264e1a..0000000
--- a/README
+++ /dev/null
@@ -1,24 +0,0 @@
-# Requirements:
-With virtualenv:
- apt install libxml2-dev libxslt1-dev python-dev libmysqlclient-dev
- virtualenv venv
- . venv/bin/activate
- pip install -r requirements.txt
-
-Without venv:
- python-twisted python-mysqldb python-lxml python-openssl python-requests
-
-$ python snurr.py -h
-Usage: python snurr.py [options] CHANNEL
-
-Pipes UDP-messages to an IRC-channel.
-
-Options:
- -h, --help show this help message and exit
- -c SERVER, --connect=SERVER
- IRC server (default: irc.ifi.uio.no)
- -p PORT, --port=PORT IRC server port (default: 6667)
- -s, --ssl connect with SSL (default: False)
- -l LISTEN_PORT, --listen_port=LISTEN_PORT
- UDP listen port (default: 55666)
-
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..5075da1
--- /dev/null
+++ b/README.md
@@ -0,0 +1,30 @@
+Friendly IRC bot using Twisted (python 2.7)
+
+# Requirements:
+With virtualenv:
+
+ apt install libxml2-dev libxslt1-dev python-dev libmysqlclient-dev
+ virtualenv venv
+ . venv/bin/activate
+ pip install -r requirements.txt
+
+Without venv:
+
+ apt install python-twisted python-mysqldb python-lxml python-openssl python-requests
+
+Usage:
+
+ $ python snurr.py -h
+ Usage: python snurr.py [-h] [options] CHANNEL
+
+ Pipes UDP-messages to an IRC-channel.
+
+ Options:
+ -h, --help show this help message and exit
+ -c SERVER, --connect=SERVER
+ IRC server (default: irc.oftc.net)
+ -p PORT, --port=PORT IRC server port (default: 6697)
+ -s, --ssl connect with SSL (default: False)
+ -l LISTEN_PORT, --listen_port=LISTEN_PORT
+ UDP listen port (default: 55666)
+
diff --git a/requirements.txt b/requirements.txt
index 05184ca..3b9f38d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,6 @@
-Twisted[tls]==16.0.0
+Twisted[tls]==16.2.0
lxml==3.6.0
+micawber==0.3.3
mysqlclient==1.3.7
-requests==2.9.1
+requests==2.10.0
ndg-httpsclient==0.4.0
\ No newline at end of file
diff --git a/snurr.py b/snurr.py
index b223cd4..b660422 100755
--- a/snurr.py
+++ b/snurr.py
@@ -1,7 +1,6 @@
#!/usr/bin/env python
# encoding: utf-8
import optparse
-import requests
import re
import logging
from twisted.words.protocols import irc
@@ -9,7 +8,7 @@ from twisted.internet import protocol, reactor, ssl
import settings
-from utils import ReconnectingConnectionPool, ping_host, parse_title
+from utils import ReconnectingConnectionPool, ping_host, get_reply_from_url
logger = logging.getLogger('snurr')
@@ -157,27 +156,10 @@ class IRCActions:
# Check if msg contains HTTP(S) URL
urlmatch = re.search(r"(https?):\/\/([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w\.-]*\/?)(\?[\/\w=&\.-]+)?", msg, re.I)
if urlmatch is not None:
- self.msg_urlinfo(urlmatch, channel, nick)
-
- def msg_urlinfo(self, urlmatch, channel, nick):
- reply = "URL: "
- url = urlmatch.group()
- try:
- req = requests.get(url, headers={'User-Agent': 'snurrbot v0.1'}, stream=True)
- except Exception as e:
- logger.info(e)
- self.bot.msgReply(nick, channel, reply + "Timeout")
- return
-
- content_type = req.headers['content-type']
- if ';' in content_type:
- content_type = content_type.split(';')[0]
- reply += "%d %s " % (req.status_code, content_type)
- if req.status_code == 200 and content_type == "text/html":
- # Read max 20480 bytes
- partial_html = next(req.iter_content(chunk_size=20480))
- reply += parse_title(partial_html)
+ self.msg_urlinfo(urlmatch.group(), channel, nick)
+ def msg_urlinfo(self, url, channel, nick):
+ reply = get_reply_from_url(url)
self.bot.msgReply(nick, channel, reply)
def get_tetris_highscore(self):
diff --git a/tests.py b/tests.py
index 0ec0586..525a2a8 100644
--- a/tests.py
+++ b/tests.py
@@ -1,7 +1,7 @@
+from __future__ import unicode_literals
import unittest
-import requests
-from utils import parse_title
+from utils import parse_title, get_reply_from_url
class ParseHTMLTest(unittest.TestCase):
@@ -12,14 +12,12 @@ class ParseHTMLTest(unittest.TestCase):
{}
""".format(title)
- self.assertEqual(parse_title(partial_html=partial_html), '[{}]'.format(title))
+ self.assertEqual(parse_title(html=partial_html), title)
- # @unittest.skip()
- def test_parse_title_from_stream(self):
- url = 'http://lol.com'
- req = requests.get(url, headers={'User-Agent': 'snurrbot v0.1'}, stream=True)
- partial_html = req.iter_content(chunk_size=10240)
- self.assertEqual(parse_title(partial_html=partial_html), None)
+ def test_get_reply_from_url(self):
+ url = 'https://www.youtube.com/watch?v=3yR_BAqB7aQ'
+ reply = get_reply_from_url(url)
+ self.assertEqual(reply, 'URL: 200 text/html [Daydream Labs: Puzzle - YouTube]')
if __name__ == '__main__':
unittest.main()
diff --git a/utils.py b/utils.py
index 99cd18c..139cb61 100644
--- a/utils.py
+++ b/utils.py
@@ -1,8 +1,12 @@
-import logging
-import re
-import subprocess
+from __future__ import unicode_literals
+import logging
import lxml.html
+from lxml.etree import XMLSyntaxError
+import micawber
+import re
+import requests
+import subprocess
from twisted.enterprise import adbapi
@@ -53,22 +57,55 @@ def ping_host(host):
return "feil med ping:" + str(e)
-def parse_title(partial_html):
- reply = ''
+def get_title_with_oembed(url):
+ providers = micawber.bootstrap_basic()
try:
- # This might raise an exception if the HTML is _really_ broken
- tree = lxml.html.fromstring(partial_html)
- title = tree.find(".//title")
- if title is not None and title.text is not None:
- reply += "[%s]" % re.sub(r'\s+', ' ', title.text.strip())
- # DEBUG
- if title is None:
- reply += ""
- elif title.text is None:
- reply += "<title.text is none>"
- # END
- except Exception as e:
- reply += "<exception>"
- logger.error(e.message)
+ res = providers.request(url)
+ except micawber.ProviderException as e:
+ logger.info(e)
+ return ''
+
+ return '{} - {}'.format(res.get('title'), res.get('provider_name'))
+
+
+def get_reply_from_url(url):
+ user_agent = 'snurrbot v0.1'
+
+ # Try using oembed
+ title = get_title_with_oembed(url)
+
+ # Fetch the html
+ try:
+ req = requests.get(url, headers={'User-Agent': user_agent})
+ except requests.Timeout as e:
+ logger.info(e)
+ return 'URL Timeout'
+
+ content_type = req.headers['content-type']
+ if ';' in content_type:
+ content_type = content_type.split(';')[0]
+
+ if not title and req.status_code == 200 and content_type == "text/html":
+ title = parse_title(req.text)
+
+ title = '' if not title else ' [{}]'.format(title)
+ reply = "URL: {} {}{}".format(req.status_code, content_type, title)
return reply
+
+
+def parse_title(html):
+ title = ''
+
+ try:
+ # This might raise an exception if the HTML is _really_ broken
+ tree = lxml.html.fromstring(html)
+ title_tag = tree.find(".//title")
+
+ if title_tag is not None and title_tag.text is not None:
+ title += "%s" % re.sub(r'\s+', ' ', title_tag.text.strip())
+
+ except XMLSyntaxError as e:
+ logger.info(str(e))
+
+ return title