0.6.1

HISTORY.rst

@@ -100,3 +100,8 @@ Main differences with last major release:
    improvements were added on the last versions of 0.5.x
  * Tests have been added that cover most of the code in tweets.py and utils.py.
    In the next versions probably even more will be added.
+Version 0.6.1:
+Added more tests and the "as_json()" function divided in two: "as_long_json",
+which simply returns the __dict__ of the Tweet instance, with all its key-value
+pairs, and "as_short_json", which returns just the non-empty key-value pairs
+(and it won't even return nested dicts it they have no content).

requirements.txt

@@ -34,6 +34,7 @@ PyGithub==1.43.5
 Pygments==2.3.1
 PyJWT==1.7.1
 pyparsing==2.3.1
+pysnooper==0.0.38
 python-dateutil==2.8.0
 python-gitlab==1.8.0
 pytz==2018.9

setup.py

@@ -51,6 +51,6 @@ setup(
     test_suite='tests',
     tests_require=test_requirements,
     url='https://github.com/Serbaf/tweet_model',
-    version='0.6.0',
+    version='0.6.1',
     zip_safe=False,
 )

tests/test_tweet.py

@@ -9,12 +9,12 @@ import datetime
 #################
 #   FIXTURES    #
 #################
-@pytest.fixture
+@pytest.fixture(scope="module")
 def empty_tweet():
     return Tweet()
 
 
-@pytest.fixture
+@pytest.fixture(scope="module")
 def tweet1():
     return Tweet(id="128977963458933",
                  text="En un lugar de la Mancha...",
@@ -22,7 +22,22 @@ def tweet1():
                  user__created_at="2019-05-01 23:54:27")
 
 
-@pytest.fixture
+@pytest.fixture(scope="module")
+def tweet2(csv_file_path):
+    """
+    Returns a real tweet instantiated from a CSV. For this to work
+    test_get_tweet_from_csv_raw_line_OK should be OK. The instantiated tweet is
+    the first in the CSV doc (2nd line, just after the header)
+    """
+
+    with open(csv_file_path) as file_reader:
+        header = file_reader.readline()
+        tweet_contents = file_reader.readline()
+    tweet = utils.get_tweet_from_csv_raw_line(header, tweet_contents)
+    return tweet
+
+
+@pytest.fixture(scope="module")
 def dict_tweet1():
     dictionary_tweet1 = {}
     dictionary_tweet1["id"] = 128977963458933
@@ -35,13 +50,60 @@ def dict_tweet1():
     return dictionary_tweet1
 
 
-@pytest.fixture
+@pytest.fixture(scope="module")
+def dict_tweet2():
+    """
+    Dictionary representation of the same data as the fixture "tweet2"
+    """
+
+    dictionary_tweet2 = {}
+    dictionary_tweet2["created_at"] = datetime.datetime(2019, 5, 1, 23, 59, 16)
+    dictionary_tweet2["favorite_count"] = 0
+    dictionary_tweet2["id"] = 1123738691938193410
+    dictionary_tweet2["in_reply_to_screen_name"] = 'leoffmiranda'
+    dictionary_tweet2["in_reply_to_status_id"] = 1123737830554066945
+    dictionary_tweet2["in_reply_to_user_id"] = 176244402
+    dictionary_tweet2["is_quote_status"] = False
+    dictionary_tweet2["retweet_count"] = 0
+    dictionary_tweet2["source"] =\
+        '<a href="http://twitter.com" rel="nofollow">Twitter Web Client</a>'
+    dictionary_tweet2["text"] = '@leoffmiranda perfeito. Pele ta na categoria Tolkien pra fantasia, pode ate existir escritores de fantasia melhores… https://t.co/sl5Nd82l71'
+    dictionary_tweet2["truncated"] = True
+
+    dictionary_tweet2["entities"] = {}
+    dictionary_tweet2["entities"]["urls"] = {}
+    dictionary_tweet2["entities"]["urls"]["expanded_url"] =\
+        ['https://twitter.com/i/web/status/1123738691938193410']
+
+    dictionary_tweet2["entities"]["user_mentions"] = {}
+    dictionary_tweet2["entities"]["user_mentions"]["screen_name"] = ['leoffmiranda']
+
+    dictionary_tweet2["user"] = {}
+    dictionary_tweet2["user"]["created_at"] = datetime.datetime(2009, 8, 24, 20, 21, 5)
+    dictionary_tweet2["user"]["favourites_count"] = 108774
+    dictionary_tweet2["user"]["followers_count"] = 4803
+    dictionary_tweet2["user"]["friends_count"] = 1044
+    dictionary_tweet2["user"]["geo_enabled"] = True
+    dictionary_tweet2["user"]["id"] = 68503128
+    dictionary_tweet2["user"]["lang"] = "pt"
+    dictionary_tweet2["user"]["listed_count"] = 45
+    dictionary_tweet2["user"]["location"] = "ArenaCorinthians"
+    dictionary_tweet2["user"]["name"] = 'Corinthians matou o futebol. #RipFutibas'
+    dictionary_tweet2["user"]["profile_image_url"] = 'http://pbs.twimg.com/profile_images/1099105765711929345/dz5f_SdP_normal.jpg'
+    dictionary_tweet2["user"]["screen_name"] = 'tathiane_vidal'
+    dictionary_tweet2["user"]["statuses_count"] = 177470
+    dictionary_tweet2["user"]["verified"] = False
+
+    return dictionary_tweet2
+
+
+@pytest.fixture(scope="module")
 def csv_file_path():
     current_dir = os.path.dirname(os.path.realpath(__file__))
     return os.path.join(current_dir, "resources", "01.csv")
 
 
-@pytest.fixture
+@pytest.fixture(scope="module")
 def fake_file_path():
     current_dir = os.path.dirname(os.path.realpath(__file__))
     return os.path.join(current_dir, "resources", "false.csv")
@@ -96,6 +158,40 @@ class TestTweetObjects:
         assert new_dict == {"id": 128977963458933,
                             "text": "En un lugar de la Mancha..."}
 
+    def test_tweet_list_fields_OK(self, tweet2):
+        """
+        Check that the list fields in the instantiated tweets work as expected
+        """
+
+        assert tweet2["entities"]["user_mentions"]["screen_name"] == \
+            ["leoffmiranda"]
+
+    def test_as_short_json_tweet1_OK(self, tweet1):
+        """
+        Check that the function "as_short_json" returns a correct
+        representation of the tweet in dict form and with just the non-empty
+        fields
+        """
+
+        result = {}
+        result["id"] = 128977963458933
+        result["text"] = "En un lugar de la Mancha..."
+        result["user"] = {}
+        result["user"]["name"] = "Julio César"
+        result["user"]["created_at"] =\
+            datetime.datetime(2019, 5, 1, 23, 54, 27)
+
+        assert result == tweet1.as_short_json()
+
+    def test_as_short_json_tweet2_OK(self, tweet2, dict_tweet2):
+        """
+        Check that the function "as_short_json" returns a correct
+        representation of the tweet2 in dict form and with just the non-empty
+        fields
+        """
+
+        assert tweet2.as_short_json() == dict_tweet2
+
 
 class TestTweetUtils:
     def test_get_tweet_from_csv_raw_line_OK(self, csv_file_path):

tweet_model/tweet.py

@@ -144,47 +144,52 @@ class Tweet():
         try:
             self.created_at =\
                 datetime.datetime.strptime(created_at, "%Y-%m-%d %H:%M:%S")
-        except (TypeError, ValueError):
+        except Exception as exc:
             self.created_at = created_at
 
         try:
             self.id = int(id)
-        except TypeError:
+        except Exception as exc:
             self.id = id
 
-        try:
-            self.truncated = bool(truncated)
-        except TypeError:
+        if type(truncated) is str:
+            if truncated == "True":
+                self.truncated = True
+            elif truncated == "False":
+                self.truncated = False
+        else:
             self.truncated = truncated
 
         try:
             self.in_reply_to_status_id = int(in_reply_to_status_id)
-        except TypeError:
+        except Exception as exc:
             self.in_reply_to_status_id = in_reply_to_status_id
 
         try:
             self.in_reply_to_user_id = int(in_reply_to_user_id)
-        except TypeError:
+        except Exception as exc:
             self.in_reply_to_user_id = in_reply_to_user_id
 
         try:
             self.quoted_status_id = int(quoted_status_id)
-        except TypeError:
+        except Exception as exc:
             self.quoted_status_id = quoted_status_id
 
-        try:
-            self.is_quote_status = bool(is_quote_status)
-        except TypeError:
+        if type(is_quote_status) is str:
+            if is_quote_status == "True":
+                self.is_quote_status = True
+            elif is_quote_status == "False":
+                self.is_quote_status = False
+        else:
             self.is_quote_status = is_quote_status
-
         try:
             self.retweet_count = int(retweet_count)
-        except TypeError:
+        except Exception as exc:
             self.retweet_count = retweet_count
 
         try:
             self.favorite_count = int(favorite_count)
-        except TypeError:
+        except Exception as exc:
             self.favorite_count = favorite_count
 
         self.text = text
@@ -216,48 +221,54 @@ class Tweet():
 
         try:
             self.user["id"] = int(user__id)
-        except TypeError:
+        except Exception as exc:
             self.user["id"] = user__id
 
         try:
             self.user["created_at"] = datetime.datetime.strptime(
                 user__created_at, "%Y-%m-%d %H:%M:%S")
-        except (TypeError, ValueError):
+        except Exception as exc:
             self.user["created_at"] = user__created_at
 
-        try:
-            self.user["verified"] = bool(user__verified)
-        except TypeError:
+        if type(user__verified) is str:
+            if user__verified == "True":
+                self.user["verified"] = True
+            elif user__verified == "False":
+                self.user["verified"] = False
+        else:
             self.user["verified"] = user__verified
 
         try:
             self.user["followers_count"] = int(user__followers_count)
-        except TypeError:
+        except Exception as exc:
             self.user["followers_count"] = user__followers_count
 
         try:
             self.user["friends_count"] = int(user__friends_count)
-        except TypeError:
+        except Exception as exc:
             self.user["friends_count"] = user__friends_count
 
         try:
             self.user["listed_count"] = int(user__listed_count)
-        except TypeError:
+        except Exception as exc:
             self.user["listed_count"] = user__listed_count
 
         try:
             self.user["favourites_count"] = int(user__favourites_count)
-        except TypeError:
+        except Exception as exc:
             self.user["favourites_count"] = user__favourites_count
 
         try:
             self.user["statuses_count"] = int(user__statuses_count)
-        except TypeError:
+        except Exception as exc:
             self.user["statuses_count"] = user__statuses_count
 
-        try:
-            self.user["geo_enabled"] = bool(user__geo_enabled)
-        except TypeError:
+        if type(user__geo_enabled) is str:
+            if user__geo_enabled == "True":
+                self.user["geo_enabled"] = True
+            elif user__geo_enabled == "False":
+                self.user["geo_enabled"] = False
+        else:
             self.user["geo_enabled"] = user__geo_enabled
 
         self.user["name"] = user__name
@@ -305,7 +316,7 @@ class Tweet():
             self.coordinates["coordinates"] =\
                 [float(coords)
                     for coords in json.loads(coordinates__coordinates)]
-        except (TypeError, IndexError):
+        except Exception as exc:
             self.coordinates["coordinates"] = coordinates__coordinates
 
         # Place object
@@ -328,7 +339,7 @@ class Tweet():
                     for y in range(len(coords[x]))]
                     for x in range(len(coords))]
 
-        except (TypeError, IndexError):
+        except Exception as exc:
             self.place["bounding_box"]["coordinates"] =\
                 place__bounding_box__coordinates
         self.place["bounding_box"]["type"] = place__bounding_box__type
@@ -338,8 +349,9 @@ class Tweet():
         # Entities hashtags
         self.entities["hashtags"] = {}
         try:
-            self.entities["hashtags"]["text"] = list(entities__hashtags__text)
-        except TypeError:
+            self.entities["hashtags"]["text"] =\
+                json.loads(entities__hashtags__text)
+        except Exception as exc:
             self.entities["hashtags"]["text"] = entities__hashtags__text
 
         self.entities["hashtags"]["indices"] = entities__hashtags__indices
@@ -347,8 +359,8 @@ class Tweet():
         self.entities["media"] = {}
         try:
             self.entities["media"]["media_url"] =\
-                list(entities__media__media_url)
-        except TypeError:
+                json.loads(entities__media__media_url)
+        except Exception as exc:
             self.entities["media"]["media_url"] = entities__media__media_url
         self.entities["media"]["display_url"] = entities__media__display_url
         self.entities["media"]["expanded_url"] = entities__media__expanded_url
@@ -397,8 +409,8 @@ class Tweet():
         self.entities["urls"] = {}
         try:
             self.entities["urls"]["expanded_url"] =\
-                list(entities__urls__expanded_url)
-        except TypeError:
+                json.loads(entities__urls__expanded_url)
+        except Exception as exc:
             self.entities["urls"]["expanded_url"] =\
                 entities__urls__expanded_url
         self.entities["urls"]["display_url"] = entities__urls__display_url
@@ -417,8 +429,8 @@ class Tweet():
         self.entities["user_mentions"] = {}
         try:
             self.entities["user_mentions"]["screen_name"] =\
-                list(entities__user_mentions__screen_name)
-        except TypeError:
+                json.loads(entities__user_mentions__screen_name)
+        except Exception as exc:
             self.entities["user_mentions"]["screen_name"] =\
                 entities__user_mentions__screen_name
         self.entities["user_mentions"]["id"] = entities__user_mentions__id
@@ -494,18 +506,44 @@ class Tweet():
                 pass
         return tweet_subset
 
-    def as_json(self) -> Dict:
+    def as_short_json(self, dictionary: Dict=None) -> Dict:
         """
-        Return the Tweet object in a JSON-like representation (nested dicts)
+        Return the Tweet object in a short JSON-like representation
+        but without all the null key-value pairs
         """
 
+        if dictionary is None:
             json_tweet = {}
-
             for key, value in self.__dict__.items():
-            if value is not None:
+                if type(value) is dict:
+                    nested_dict = self.as_short_json(value)
+                    if nested_dict is not None:
+                        json_tweet[key] = nested_dict
+                elif value is not None:
                     json_tweet[key] = value
-
             return json_tweet
 
+        else:
+            dictie = {}
+            for key, value in dictionary.items():
+                if type(value) is dict:
+                    nested_dict = self.as_short_json(value)
+                    if nested_dict is not None:
+                        dictie[key] = nested_dict
+                elif value is not None:
+                    dictie[key] = value
+            if len(dictie) == 0:
+                return None
+            else:
+                return dictie
+
+    def as_long_json(self) -> Dict:
+        """
+        Return the Tweet object in a JSON-like representation (nested dicts).
+        Just the __dict__ of the class
+        """
+
+        return self.__dict__
+
     def __getitem__(self, key):
         return getattr(self, key)