tweetarchiver: log when stream is closed by the server. set socket ti…

…meout, and reconnect upon timeout (default 30min).
internetarchive · Apr 10, 2017 · 5e39215 · 5e39215
1 parent 8ec87a3
commit 5e39215
Showing 1 changed file with 7 additions and 2 deletions.
diff --git a/twitter/tweetarchiver/tweetstream.py b/twitter/tweetarchiver/tweetstream.py
@@ -64,6 +64,8 @@ def __init__(self, path, configuration):
             "twitter_stream_scheme", "https")
         self._twitter_stream_port = self._get_configuration_key(
             "twitter_stream_port", 443)
+        self._timeout = self._get_configuration_key(
+            "twitter_stream_timeout", 1800)
 
         parts = urlparse.urlparse(path)
         self._path = parts.path
@@ -149,6 +151,7 @@ def next(self):
             try:
                 tweet = self._readline()
                 if tweet == '':
+                    self._log.info('stream closed by the server')
                     self._response = None
                     continue
 
@@ -166,10 +169,12 @@ def _open_twitter_stream(self):
         """ Creates the client and watches stream """
         if self._twitter_stream_scheme == 'https':
             http = httplib.HTTPSConnection(self._twitter_stream_host,
-                                           self._twitter_stream_port)
+                                           self._twitter_stream_port,
+                                           timeout=self._timeout)
         else:
             http = httplib.HTTPConnection(self._twitter_stream_host,
-                                          self._twitter_stream_port)
+                                          self._twitter_stream_port,
+                                          timeout=self._timeout)
         parameters = dict(self._parameters,
                           oauth_token=self._token.key,
                           oauth_consumer_key=self._consumer.key,