Oembed is nice, download more html

README

@@ -1,24 +0,0 @@
-# Requirements:
-With virtualenv:
-    apt install libxml2-dev libxslt1-dev python-dev libmysqlclient-dev
-    virtualenv venv
-    . venv/bin/activate
-    pip install -r requirements.txt
-
-Without venv:
-    python-twisted python-mysqldb python-lxml python-openssl python-requests
-
-$ python snurr.py -h
-Usage: python snurr.py [options] CHANNEL
-
-Pipes UDP-messages to an IRC-channel.
-
-Options:
-  -h, --help            show this help message and exit
-  -c SERVER, --connect=SERVER
-                        IRC server (default: irc.ifi.uio.no)
-  -p PORT, --port=PORT  IRC server port (default: 6667)
-  -s, --ssl             connect with SSL (default: False)
-  -l LISTEN_PORT, --listen_port=LISTEN_PORT
-                        UDP listen port (default: 55666)
-

README.md

@@ -0,0 +1,30 @@
+Friendly IRC bot using Twisted (python 2.7)
+
+# Requirements:
+With virtualenv:
+
+    apt install libxml2-dev libxslt1-dev python-dev libmysqlclient-dev
+    virtualenv venv
+    . venv/bin/activate
+    pip install -r requirements.txt
+
+Without venv:
+
+    apt install python-twisted python-mysqldb python-lxml python-openssl python-requests
+
+Usage:
+
+    $ python snurr.py -h
+    Usage: python snurr.py [-h] [options] CHANNEL
+
+    Pipes UDP-messages to an IRC-channel.
+
+    Options:
+      -h, --help            show this help message and exit
+      -c SERVER, --connect=SERVER
+                            IRC server (default: irc.oftc.net)
+      -p PORT, --port=PORT  IRC server port (default: 6697)
+      -s, --ssl             connect with SSL (default: False)
+      -l LISTEN_PORT, --listen_port=LISTEN_PORT
+                            UDP listen port (default: 55666)
+

requirements.txt

@@ -1,5 +1,6 @@
-Twisted[tls]==16.0.0
+Twisted[tls]==16.2.0
 lxml==3.6.0
+micawber==0.3.3
 mysqlclient==1.3.7
-requests==2.9.1
+requests==2.10.0
 ndg-httpsclient==0.4.0

snurr.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 # encoding: utf-8
 import optparse
-import requests
 import re
 import logging
 from twisted.words.protocols import irc
@@ -9,7 +8,7 @@ from twisted.internet import protocol, reactor, ssl
 
 import settings
 
-from utils import ReconnectingConnectionPool, ping_host, parse_title
+from utils import ReconnectingConnectionPool, ping_host, get_reply_from_url
 
 logger = logging.getLogger('snurr')
 
@@ -157,27 +156,10 @@ class IRCActions:
         # Check if msg contains HTTP(S) URL
         urlmatch = re.search(r"(https?):\/\/([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w\.-]*\/?)(\?[\/\w=&\.-]+)?", msg, re.I)
         if urlmatch is not None:
-            self.msg_urlinfo(urlmatch, channel, nick)
+            self.msg_urlinfo(urlmatch.group(), channel, nick)
-
-    def msg_urlinfo(self, urlmatch, channel, nick):
-        reply = "URL: "
-        url = urlmatch.group()
-        try:
-            req = requests.get(url, headers={'User-Agent': 'snurrbot v0.1'}, stream=True)
-        except Exception as e:
-            logger.info(e)
-            self.bot.msgReply(nick, channel, reply + "Timeout")
-            return
-
-        content_type = req.headers['content-type']
-        if ';' in content_type:
-            content_type = content_type.split(';')[0]
-        reply += "%d %s " % (req.status_code, content_type)
-        if req.status_code == 200 and content_type == "text/html":
-            # Read max 20480 bytes
-            partial_html = next(req.iter_content(chunk_size=20480))
-            reply += parse_title(partial_html)
 
+    def msg_urlinfo(self, url, channel, nick):
+        reply = get_reply_from_url(url)
         self.bot.msgReply(nick, channel, reply)
 
     def get_tetris_highscore(self):

tests.py

@@ -1,7 +1,7 @@
+from __future__ import unicode_literals
 import unittest
 
-import requests
+from utils import parse_title, get_reply_from_url
-from utils import parse_title
 
 
 class ParseHTMLTest(unittest.TestCase):
@@ -12,14 +12,12 @@ class ParseHTMLTest(unittest.TestCase):
             <title>{}</title>
             <meta charset="utf-8" />
             <meta name="idk" content="test" />""".format(title)
-        self.assertEqual(parse_title(partial_html=partial_html), '[{}]'.format(title))
+        self.assertEqual(parse_title(html=partial_html), title)
 
-    # @unittest.skip()
+    def test_get_reply_from_url(self):
-    def test_parse_title_from_stream(self):
+        url = 'https://www.youtube.com/watch?v=3yR_BAqB7aQ'
-        url = 'http://lol.com'
+        reply = get_reply_from_url(url)
-        req = requests.get(url, headers={'User-Agent': 'snurrbot v0.1'}, stream=True)
+        self.assertEqual(reply, 'URL: 200 text/html [Daydream Labs: Puzzle - YouTube]')
-        partial_html = req.iter_content(chunk_size=10240)
-        self.assertEqual(parse_title(partial_html=partial_html), None)
 
 if __name__ == '__main__':
     unittest.main()

utils.py

@@ -1,8 +1,12 @@
-import logging
+from __future__ import unicode_literals
-import re
-import subprocess
 
+import logging
 import lxml.html
+from lxml.etree import XMLSyntaxError
+import micawber
+import re
+import requests
+import subprocess
 from twisted.enterprise import adbapi
 
 
@@ -53,22 +57,55 @@ def ping_host(host):
         return "feil med ping:" + str(e)
 
 
-def parse_title(partial_html):
+def get_title_with_oembed(url):
-    reply = ''
+    providers = micawber.bootstrap_basic()
     try:
-        # This might raise an exception if the HTML is _really_ broken
+        res = providers.request(url)
-        tree = lxml.html.fromstring(partial_html)
+    except micawber.ProviderException as e:
-        title = tree.find(".//title")
+        logger.info(e)
-        if title is not None and title.text is not None:
+        return ''
-            reply += "[%s]" % re.sub(r'\s+', ' ', title.text.strip())
+
-        # DEBUG
+    return '{} - {}'.format(res.get('title'), res.get('provider_name'))
-        if title is None:
+
-            reply += "<title is none>"
+
-        elif title.text is None:
+def get_reply_from_url(url):
-            reply += "<title.text is none>"
+    user_agent = 'snurrbot v0.1'
-        # END
+
-    except Exception as e:
+    # Try using oembed
-        reply += "<exception>"
+    title = get_title_with_oembed(url)
-        logger.error(e.message)
+
+    # Fetch the html
+    try:
+        req = requests.get(url, headers={'User-Agent': user_agent})
+    except requests.Timeout as e:
+        logger.info(e)
+        return 'URL Timeout'
+
+    content_type = req.headers['content-type']
+    if ';' in content_type:
+        content_type = content_type.split(';')[0]
+
+    if not title and req.status_code == 200 and content_type == "text/html":
+        title = parse_title(req.text)
+
+    title = '' if not title else ' [{}]'.format(title)
+    reply = "URL: {} {}{}".format(req.status_code, content_type, title)
 
     return reply
+
+
+def parse_title(html):
+    title = ''
+
+    try:
+        # This might raise an exception if the HTML is _really_ broken
+        tree = lxml.html.fromstring(html)
+        title_tag = tree.find(".//title")
+
+        if title_tag is not None and title_tag.text is not None:
+            title += "%s" % re.sub(r'\s+', ' ', title_tag.text.strip())
+
+    except XMLSyntaxError as e:
+        logger.info(str(e))
+
+    return title