Commit 7ff9ad26 by serpucga

Added proper logging

parent 866da8f7
......@@ -6,6 +6,10 @@ from math import ceil
from typing import List
from tweet_manager.lib import json2csv, format_csv
# Logger
import logging
logger = logging.getLogger(__name__)
def write_tweets_to_files(
host: str,
......@@ -16,7 +20,7 @@ def write_tweets_to_files(
output_dir: str,
page_index: List[int])\
-> None:
print("Hi there! write_tweets_to_files executing")
logger.info("Hi there! write_tweets_to_files executing")
client = pymongo.MongoClient(host, port)
database_tweets = client[database]["tweets"]
tweets_page = get_tweets_page(database_tweets, pagesize, page_index)
......@@ -108,14 +112,14 @@ def create_task_database_structure(
# Create the root directory for the tweet collection
(output_dir, db_name) = os.path.split(output_dir)
if not os.path.isdir(output_dir):
print(
logger.info(
"Building directory to contain the collected tweets at: "
+ os.path.abspath(output_dir)
)
os.mkdir(output_dir)
collection_path = os.path.join(output_dir, db_name)
if not os.path.isdir(collection_path):
print("Initializing collection " + db_name + "...")
logger.info("Initializing collection " + db_name + "...")
os.mkdir(collection_path)
return collection_path
......
......@@ -3,6 +3,7 @@
import pymongo
import os
import argparse
import logging
import multiprocessing as mp
from lib import utils
......@@ -12,9 +13,20 @@ parser = argparse.ArgumentParser(
parser.add_argument("-H", "--host", type=str, default="localhost")
parser.add_argument("-p", "--port", type=int, default=27017)
parser.add_argument("-s", "--pagesize", type=int, default=1000)
parser.add_argument("-v", "--verbose", action="store_true")
parser.add_argument("database", type=str)
args = parser.parse_args()
# Activate logging
logformat = "[%(asctime)s]: %(message)s"
dateformat = "%H:%M:%S"
if args.verbose:
logging.basicConfig(
level=logging.DEBUG, format=logformat, datefmt=dateformat)
else:
logging.basicConfig(
level=logging.ERROR, format=logformat, datefmt=dateformat)
# Initialize some variables
script_dir = os.path.dirname(__file__)
output_dir = os.path.join(script_dir, "pymongodump", args.database)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment