Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
M
migration_scripts
Project
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Registry
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
serpucga
migration_scripts
Commits
4516d8ee
Commit
4516d8ee
authored
Jul 25, 2019
by
serpucga
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Exit appropriately when there are no tweets in the collection
parent
7441fb7f
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
25 additions
and
2 deletions
+25
-2
utils.py
lib/utils.py
+15
-1
pymongoexport_csv.py
pymongoexport_csv.py
+10
-1
No files found.
lib/utils.py
View file @
4516d8ee
...
@@ -416,7 +416,13 @@ def get_page_index_fast(
...
@@ -416,7 +416,13 @@ def get_page_index_fast(
page
=
collection
.
find
({
"id"
:
{
"$lt"
:
last_id
}})
\
page
=
collection
.
find
({
"id"
:
{
"$lt"
:
last_id
}})
\
.
sort
(
"id"
,
pymongo
.
DESCENDING
)
\
.
sort
(
"id"
,
pymongo
.
DESCENDING
)
\
.
limit
(
page_size
)
.
limit
(
page_size
)
pages
.
append
(
page
[
0
][
"id"
])
try
:
pages
.
append
(
page
[
0
][
"id"
])
except
IndexError
as
exc
:
if
collection
.
count
()
==
0
:
raise
NoTweetsException
else
:
raise
exc
try
:
try
:
last_id
=
page
[
page_size
-
1
][
"id"
]
last_id
=
page
[
page_size
-
1
][
"id"
]
except
IndexError
:
except
IndexError
:
...
@@ -525,3 +531,11 @@ class TweetConversionException(Exception):
...
@@ -525,3 +531,11 @@ class TweetConversionException(Exception):
"""
"""
self
.
message
=
message
self
.
message
=
message
self
.
tweet
=
tweet
self
.
tweet
=
tweet
class
NoTweetsException
(
Exception
):
"""
Should be raised when not even 1 tweet is found in the collection
"""
pass
pymongoexport_csv.py
View file @
4516d8ee
...
@@ -71,7 +71,16 @@ if args.recovery:
...
@@ -71,7 +71,16 @@ if args.recovery:
else
:
else
:
client
=
pymongo
.
MongoClient
(
args
.
host
,
args
.
port
)
client
=
pymongo
.
MongoClient
(
args
.
host
,
args
.
port
)
database_tweets
=
client
[
args
.
database
][
"tweets"
]
database_tweets
=
client
[
args
.
database
][
"tweets"
]
page_index
=
utils
.
get_page_index_fast
(
database_tweets
,
args
.
pagesize
)
try
:
page_index
=
utils
.
get_page_index_fast
(
database_tweets
,
args
.
pagesize
)
except
utils
.
NoTweetsException
:
logger
.
error
(
"Collection has no tweets"
)
os
.
system
(
'spd-say "Collection has no tweets"'
)
sys
.
exit
(
0
)
except
Exception
:
os
.
system
(
"spd-say 'Something really bad happened!'"
)
sys
.
exit
(
1
)
client
.
close
()
client
.
close
()
logger
.
debug
(
logger
.
debug
(
"Database {} partitioned in {} pages of {} tweets (maximum)"
"Database {} partitioned in {} pages of {} tweets (maximum)"
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment