Skip to content

Commit

Permalink
Merge branch 'live-mabed' of https://github.com/AdrienGuille/MABED in…
Browse files Browse the repository at this point in the history 
…to live-mabed
  • Loading branch information
@nicolasdugue
nicolasdugue committed Feb 11, 2016
2 parents af66fdb + e5c96e1 commit 1ece477
Show file tree
Hide file tree
Showing 2 changed files with 114 additions and 109 deletions.
29 changes: 11 additions & 18 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ Finally, the *orchestrator* jar can be used to handle all the previous jars effi
###Obtaining Twitter Tokens
----------------------

Twitter tokens are mandatory to use MABED and Live-MABED. Thay allow the application to get data from Twitter.
Twitter tokens are mandatory to use Live-MABED. Thay allow the application to get data from Twitter.
To get usable tokens, please follow this link : [https://dev.twitter.com/oauth/overview/application-owner-access-tokens] and especially the section introduced by "At the bottom of the next page, you will see a section labeled “your access token”:".

###Running the program
Expand All @@ -54,31 +54,24 @@ To get usable tokens, please follow this link : [https://dev.twitter.com/oauth/o
-c,--consumer <arg> Consumer key
-cs,--consumerkey <arg> Secret Consumer key
-e,--exp <arg> Experiment Name : ONE WORD ONLY
-geo,--geolocation <arg> Coordinates (long lat long lat) of the bouding
box to use to filter the tweet stream
-h,--help print this message
-k,--events <arg> Number of events to detect. Default to 20.
-keyword,--keyword <arg> Keywords to use to filter the tweet stream
-m,--minutes <arg> Time interval in minutes. Default : 30.
-lang,--language <arg> Language of the tweets to collect. Default to
'en'
-m,--minutes <arg> Time interval in minutes. Default: 30.
-ms,--minsupport <arg> Parameter for keyword selection between 0 and
1. Default to 0.01
1. Default to 0.01
-Ms,--maxsupport <arg> Parameter for keyword selection between 0 and
1. Default to 0.1
1. Default to 0.1
-nt,--thread <arg> Number of Threads
-p,--keywords <arg> Number of keywords per event. Default to 10.
-p,--nbkeywords <arg> Number of keywords per event. Default to 10.
-period,--period <arg> How many time intervals make a period.
-sigma,--sigma <arg> Parameter to control event redundancy between
0 and 1. Default to 0.5
0 and 1. Default to 0.5
-t,--token <arg> Twitter token
-theta,--theta <arg> Parameter for keyword selection between 0 and
1. Default to 0.7
1. Default to 0.7
-ts,--secrettoken <arg> Secret Twitter token


###Files in the Directory
----------------------

- input/: input files that describe the corpus in which we want to detect events
- MABED.jar: Java program that does the event detection
- README.txt: this file
- parameters.txt: Java properties file in which parameters are set
- stopwords.txt: a list of common stopwords to remove when generating the vocabulary
- lib/: program dependencies
194 changes: 103 additions & 91 deletions src/fr/loria/twitterstream/Streaming.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,21 +22,20 @@
import twitter4j.TwitterStreamFactory;
import twitter4j.auth.AccessToken;


/**
* @author Nicolas Dugué
*
* Main Class used to collect tweets
* Main Class used to collect tweets
*/
public class Streaming {

private static HelpFormatter formatter = new HelpFormatter();
private static CommandLineParser parser = new DefaultParser();
private static Options options = new Options();
private static Logger log = AppLogger.getInstance();

/**
* Allows to create the Streaming Command Line Interface
* Allows to create the Streaming Command Line Interface
*/
public static void createCLI() {
options.addOption("h", "help", false, "print this message");
Expand All @@ -50,147 +49,160 @@ public static void createCLI() {
options.addOption(option);
option = new Option("k", "keyword", true, "Keywords to use to filter the tweet stream");
options.addOption(option);
option = new Option("geo", "geolocation", true, "Coordinates of the bounding box to use to filter the tweet stream");
option = new Option("geo", "geolocation", true,
"Coordinates of the bounding box to use to filter the tweet stream");
options.addOption(option);
option = new Option("lang", "language", true, "Language of the tweets to collect. Default to 'en'");
option = new Option("lang", "language", true, "Language of the tweets to collect. Default to 'en'");
options.addOption(option);
option = new Option("e", "exp", true, "Experiment Name : ONE WORD ONLY");
options.addOption(option);
option = new Option("m", "minutes", true, "Time interval in minutes. Default : 30.");
options.addOption(option);
}

/**
* Allows to check whether or not the parameters necessary to run the stream collection were correctly filled
* Allows to check whether or not the parameters necessary to run the stream
* collection were correctly filled
*
* @param line The Command Line interface with its options
* @param line
* The Command Line interface with its options
* @return true if the parameters are correct
*/
public static boolean check(CommandLine line) {
boolean okay=true;
boolean okay = true;
if (!line.hasOption("ts")) {
log.warn("You need to provide a secret token");
okay=false;
okay = false;
}
if (!line.hasOption("t")) {
log.warn("You need to provide a token");
okay=false;
okay = false;
}
if (!line.hasOption("c")) {
log.warn("You need to provide a consumer key");
okay=false;
okay = false;
}
if (!line.hasOption("cs")) {
log.warn("You need to provide a secret consumer key");
okay=false;
okay = false;
}
if (!line.hasOption("k") && !line.hasOption("geo")) {
log.warn("You need to provide either keywords or coordinates");
okay=false;
okay = false;
}
if (!line.hasOption("e")) {
log.warn("You need to provide an experiment name : ONE WORD ONLY");
okay=false;
okay = false;
}
if (!okay)
printHelp();
return okay;

}

/**
* Print the Command Line Interface Help
*/
public static void printHelp() {
formatter.printHelp("Streaming API", options);
}
public static void main(String[] args) throws TwitterException, IOException, ParseException{
//Create the Command Line Interface

public static void main(String[] args) throws TwitterException, IOException, ParseException {
// Create the Command Line Interface
createCLI();
//Parse the arguments filled by the user
// Parse the arguments filled by the user
CommandLine line = parser.parse(options, args);
//Check if the arguments were correctly filled
// Check if the arguments were correctly filled
boolean process = check(line);
if (process) {
//Get the arguments
// Get the arguments
String consumer = line.getOptionValue("c");
String consumerSecret = line.getOptionValue("cs");
String token = line.getOptionValue("t");
String tokenSecret = line.getOptionValue("ts");
String language = line.getOptionValue("lang");
String language = line.getOptionValue("lang");
String keywords = null;
if(line.hasOption("k")){
keywords = "";
line.getOptionValue("k");
for (String s : line.getArgs())
keywords += " " + s;
}
double[][] coordinates = null;
String geolocation = "";
if(line.hasOption("geo")){
geolocation = line.getOptionValue("geo");
coordinates = new double[2][2];
String[] boundingBox = geolocation.split(" ");
coordinates[0][0] = Double.parseDouble(boundingBox[0]);
coordinates[0][1] = Double.parseDouble(boundingBox[1]);
coordinates[1][0] = Double.parseDouble(boundingBox[2]);
coordinates[1][1] = Double.parseDouble(boundingBox[3]);
}
if (line.hasOption("k")) {
keywords = "";
//Fix issue #3
keywords+=line.getOptionValue("k");
for (String s : line.getArgs())
keywords += " " + s;
}
double[][] coordinates = null;
String geolocation = "";
if (line.hasOption("geo")) {
geolocation = line.getOptionValue("geo");
coordinates = new double[2][2];
String[] boundingBox = geolocation.split(" ");
coordinates[0][0] = Double.parseDouble(boundingBox[0]);
coordinates[0][1] = Double.parseDouble(boundingBox[1]);
coordinates[1][0] = Double.parseDouble(boundingBox[2]);
coordinates[1][1] = Double.parseDouble(boundingBox[3]);
}
final String exp = line.getOptionValue("e");
final int minutes;
if (line.hasOption("m"))
minutes=Integer.parseInt(line.getOptionValue("m"));
minutes = Integer.parseInt(line.getOptionValue("m"));
else
minutes=30;

//Used to log the stream
StatusListener listener = new StatusListener(){
public void onStatus(Status status) {
String s = status.getText().replace("\n", " ");
//HTMLLogger.logTweet(s, exp);
try {
MabedWriter.logTweet(s, exp);
} catch (IOException e) {
}
}
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}
public void onException(Exception ex) {
log.warn(ex.getMessage());
}
public void onScrubGeo(long arg0, long arg1) {
}
public void onStallWarning(StallWarning arg0) {
}
};
//Used to close the files when the app is shut down
//A hook thread is a thred called just before the app shut down
Runtime.getRuntime().addShutdownHook(new HookThread());
TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
//Set the consumer keys and token
twitterStream.setOAuthConsumer(consumer, consumerSecret);
AccessToken act = new AccessToken(token,tokenSecret);
twitterStream.setOAuthAccessToken(act);
twitterStream.addListener(listener);
ChangeFileThread cf = new ChangeFileThread(minutes);
Thread t = new Thread(cf);
t.start();
if(keywords != null){
log.info("Stream opened on : " + keywords);
FilterQuery fq = new FilterQuery();
fq.track(keywords);
fq.language(language);
twitterStream.filter(fq);
}else{
if(coordinates != null){
log.info("Stream opened on : " + geolocation);
FilterQuery fq = new FilterQuery();
fq.locations(coordinates);
fq.language(language);
twitterStream.filter(fq);
}else{
log.info("Stream opening failed");
}
}
minutes = 30;

// Used to log the stream
StatusListener listener = new StatusListener() {
public void onStatus(Status status) {
String s = status.getText().replace("\n", " ");
// HTMLLogger.logTweet(s, exp);
try {
MabedWriter.logTweet(s, exp);
} catch (IOException e) {
}
}

public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
}

public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
}

public void onException(Exception ex) {
log.warn(ex.getMessage());
}

public void onScrubGeo(long arg0, long arg1) {
}

public void onStallWarning(StallWarning arg0) {
}
};
// Used to close the files when the app is shut down
// A hook thread is a thred called just before the app shut down
Runtime.getRuntime().addShutdownHook(new HookThread());
TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
// Set the consumer keys and token
twitterStream.setOAuthConsumer(consumer, consumerSecret);
AccessToken act = new AccessToken(token, tokenSecret);
twitterStream.setOAuthAccessToken(act);
twitterStream.addListener(listener);
ChangeFileThread cf = new ChangeFileThread(minutes);
Thread t = new Thread(cf);
t.start();
if (keywords != null) {
log.info("Stream opened on : " + keywords);
FilterQuery fq = new FilterQuery();
fq.track(keywords);
fq.language(language);
twitterStream.filter(fq);
} else {
if (coordinates != null) {
log.info("Stream opened on : " + geolocation);
FilterQuery fq = new FilterQuery();
fq.locations(coordinates);
fq.language(language);
twitterStream.filter(fq);
} else {
log.info("Stream opening failed");
}
}
}
}
}

0 comments on commit 1ece477

Please sign in to comment.