Added proper logging

lib/utils.py

@@ -6,6 +6,10 @@ from math import ceil
 from typing import List
 from tweet_manager.lib import json2csv, format_csv
 
+# Logger
+import logging
+logger = logging.getLogger(__name__)
+
 
 def write_tweets_to_files(
         host: str,
@@ -16,7 +20,7 @@ def write_tweets_to_files(
         output_dir: str,
         page_index: List[int])\
         -> None:
-    print("Hi there! write_tweets_to_files executing")
+    logger.info("Hi there! write_tweets_to_files executing")
     client = pymongo.MongoClient(host, port)
     database_tweets = client[database]["tweets"]
     tweets_page = get_tweets_page(database_tweets, pagesize, page_index)
@@ -108,14 +112,14 @@ def create_task_database_structure(
     # Create the root directory for the tweet collection
     (output_dir, db_name) = os.path.split(output_dir)
     if not os.path.isdir(output_dir):
-        print(
+        logger.info(
             "Building directory to contain the collected tweets at: "
             + os.path.abspath(output_dir)
         )
         os.mkdir(output_dir)
     collection_path = os.path.join(output_dir, db_name)
     if not os.path.isdir(collection_path):
-        print("Initializing collection " + db_name + "...")
+        logger.info("Initializing collection " + db_name + "...")
         os.mkdir(collection_path)
 
     return collection_path

pymongoexport_csv.py

@@ -3,6 +3,7 @@
 import pymongo
 import os
 import argparse
+import logging
 import multiprocessing as mp
 from lib import utils
 
@@ -12,9 +13,20 @@ parser = argparse.ArgumentParser(
 parser.add_argument("-H", "--host", type=str, default="localhost")
 parser.add_argument("-p", "--port", type=int, default=27017)
 parser.add_argument("-s", "--pagesize", type=int, default=1000)
+parser.add_argument("-v", "--verbose", action="store_true")
 parser.add_argument("database", type=str)
 args = parser.parse_args()
 
+# Activate logging
+logformat = "[%(asctime)s]: %(message)s"
+dateformat = "%H:%M:%S"
+if args.verbose:
+    logging.basicConfig(
+        level=logging.DEBUG, format=logformat, datefmt=dateformat)
+else:
+    logging.basicConfig(
+        level=logging.ERROR, format=logformat, datefmt=dateformat)
+
 # Initialize some variables
 script_dir = os.path.dirname(__file__)
 output_dir = os.path.join(script_dir, "pymongodump", args.database)