get_following_ids.py

from keys_api1 import *
import sys
import tweepy
import time
from random import randint
from functools import reduce
import pandas as pd
import csv

#Get ids from csv file 

test = pd.read_csv('output.csv')
test = test.drop(test.columns[0], axis=1)

#Preparing the output csv file

# assign header columns
headerList = ['Account_ID', 'Following_ids']
  
# open CSV file and assign header
with open("./ids/output_ids.csv", 'w') as file:
    dw = csv.DictWriter(file, delimiter=',', 
                        fieldnames=headerList)
    dw.writeheader()

error_count = 0

#iterate over ids
for index, item in test.iterrows():
    t = time.localtime()
    current_time = time.strftime("%H:%M:%S", t)
    print("Processing of ID : ", item['twitter_ids'], " TIMESTAMP : ", current_time)
    while True:
        try:
            auth = tweepy.OAuthHandler(api_key, api_key_secret)
            auth.set_access_token(access_token, access_token_secret)
            api = tweepy.API(auth, wait_on_rate_limit=True)

            id = item['twitter_ids']

            c = tweepy.Cursor(api.get_friend_ids, user_id = id, count = 4990)
            ids = []
            for page in c.pages():
                ids.append(page)
            
            #2D list to 1D and saving to a csv output

            ids = reduce(lambda x,y :x+y ,ids)
            dict = { "id" : id, "twitter_ids" : ids}
            df = pd.DataFrame(dict)
            df.to_csv("./ids/output_ids.csv", mode='a', index = False, header= False)

            #terminate the processus
            t = time.localtime()
            current_time = time.strftime("%H:%M:%S", t)
            print("Processus finished -- ", current_time, " --")
            print("Summary : ")
            print("Index : ", index, " | Errors_count : ", error_count)
            break
        
        #Handling exception by displaying them and resuming the process
        except tweepy.TwitterServerError:
            t = time.localtime()
            current_time = time.strftime("%H:%M:%S", t)
            print("Tweepy HTTP Exception detected -- ", current_time, " --")
            print ("tweepy.HTTPException = ", tweepy.TwitterServerError)
            continue

        except tweepy.HTTPException:
            t = time.localtime()
            current_time = time.strftime("%H:%M:%S", t)
            print("Tweepy HTTP Exception detected -- ", current_time, " --")
            print ("tweepy.HTTPException = ", tweepy.HTTPException)

            #terminate the processus
            error_count += 1
            t = time.localtime()
            current_time = time.strftime("%H:%M:%S", t)
            print("Processus finished with an error -- ", current_time, " -- ")
            print("Summary : ")
            print("Index : ", index, " | Errors_count : ", error_count)
            break

        except tweepy.TweepyException:
            t = time.localtime()
            current_time = time.strftime("%H:%M:%S", t)
            print("Tweepy Exception detected -- ", current_time, " --")
            print ("tweepy.TweepyException = ", tweepy.TweepyException)
            continue

        except KeyboardInterrupt:
            t = time.localtime()
            current_time = time.strftime("%H:%M:%S", t)
            print("Raising SystemExit -- ", current_time)

            #terminate the processus
            t = time.localtime()
            current_time = time.strftime("%H:%M:%S", t)
            print("Processus finished with an error -- ", current_time, " --")

            save = []
            save.append(index)
            dict = {"index : " : save}
            index_df = pd.DataFrame(dict)
            index_df.to_csv('index.csv')

            raise SystemExit
            
        except:
            e = sys.exc_info()[0]
            t = time.localtime()
            current_time = time.strftime("%H:%M:%S", t)
            print("Unknown Exception detected -- ", current_time, " --")
            print("Error: %s" % e)
            continue

save = []
save.append(index)
dict = {"index : " : save}
index_df = pd.DataFrame(dict)
index_df.to_csv('index_final.csv')