Commit bf54c24141c7bd8013f458e291b30761dbe13e9a

Authored by Cristina Muntean
1 parent 12322748

fix bug that misplaced tweets correlated with trends; create trend from file CLI

... ... @@ -78,11 +78,12 @@
78 78 <target>1.5</target>
79 79 </configuration>
80 80 </plugin>
  81 +
81 82  
82 83 <plugin>
83 84 <groupId>org.apache.maven.plugins</groupId>
84 85 <artifactId>maven-shade-plugin</artifactId>
85   - <version>2.0</version>
  86 + <version>2.3</version>
86 87 <executions>
87 88 <execution>
88 89 <phase>package</phase>
... ...
project.properties
... ... @@ -16,10 +16,9 @@ oauth.consumerSecret=xN8yjOvh0s77Ywbcw8eTPVe0H2unWIOJhGQa314A
16 16 oauth.accessToken=227458257-v3rknWtpf1bMzgzZnEhgpcdvfTtGFVyzT0SHAQ84
17 17 oauth.accessTokenSecret=4OQZmCPUAMYZ78sONqRN4mpE2fpAuU0S7ChdLC4U
18 18  
19   -min.freq=50
  19 +min.freq=15
20 20  
21 21 #bucket interval in minutes
22   -bucket.interval=30
23   -
  22 +bucket.interval=20
24 23  
25 24  
... ...
scripts/trends-from-files.sh 0 โ†’ 100755
  1 +#!/usr/bin/env bash
  2 +source ./scripts/config.sh
  3 +
  4 +EXPECTED_ARGS=2
  5 +
  6 +if [ $# -ne $EXPECTED_ARGS ]
  7 +then
  8 + echo "Usage: `basename $0` twitter-dumps-directory trends-from-file.json"
  9 + exit $E_BADARGS
  10 +fi
  11 +
  12 +echo "computing trends"
  13 +$JAVA $CLI.FileTrendsCLI -input $1 -output $2
... ...
src/main/java/it/cnr/isti/hpc/trends/cli/FileTrendsCLI.java 0 โ†’ 100644
  1 +/**
  2 + * Copyright 2015 Diego Ceccarelli
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package it.cnr.isti.hpc.trends.cli;
  17 +
  18 +import java.io.File;
  19 +
  20 +import it.cnr.isti.hpc.cli.AbstractCommandLineInterface;
  21 +import it.cnr.isti.hpc.property.ProjectProperties;
  22 +import it.cnr.isti.hpc.trends.TrendDetector;
  23 +import it.cnr.isti.hpc.trends.listener.TwitterAPIListener;
  24 +import it.cnr.isti.hpc.trends.listener.TwitterFileListener;
  25 +import it.cnr.isti.hpc.trends.output.ConsoleCollector;
  26 +import it.cnr.isti.hpc.trends.output.FileTrendCollector;
  27 +import it.cnr.isti.hpc.trends.output.ResiltechCollector;
  28 +
  29 +import javax.activity.InvalidActivityException;
  30 +
  31 +import org.slf4j.Logger;
  32 +import org.slf4j.LoggerFactory;
  33 +
  34 +import ch.qos.logback.core.ConsoleAppender;
  35 +
  36 +/**
  37 + * @author Diego Ceccarelli <diego.ceccarelli@isti.cnr.it>
  38 + *
  39 + * Created on Mar 4, 2015
  40 + */
  41 +public class FileTrendsCLI extends AbstractCommandLineInterface {
  42 +
  43 + static ProjectProperties properties = new ProjectProperties(
  44 + FileTrendsCLI.class);
  45 +
  46 + private static final Logger logger = LoggerFactory
  47 + .getLogger(FileTrendsCLI.class);
  48 +
  49 + public FileTrendsCLI(String[] args) {
  50 + super(args, new String[] {INPUT , OUTPUT});
  51 + }
  52 +
  53 + public static void main(String[] args) throws InvalidActivityException {
  54 + FileTrendsCLI cli = new FileTrendsCLI(args);
  55 + int minFreq = properties.getInt("min.freq");
  56 + int bucketInterval = properties.getInt("bucket.interval");
  57 + logger.info("term minimum frequency = {}", minFreq);
  58 + logger.info("trend detected every {} minutes", bucketInterval);
  59 +
  60 + TrendDetector td = new TrendDetector().setMinFreq(minFreq)
  61 + .setBucketInterval(bucketInterval);
  62 + td.addCollector(new FileTrendCollector(cli.getOutput()+".tsv")
  63 + .asPlain());
  64 + td.addCollector(new FileTrendCollector(cli.getOutput()+".json"));
  65 + TwitterFileListener listener = new TwitterFileListener(td);
  66 + listener.setTarget(new File(cli.getInput()));
  67 +
  68 + listener.run();
  69 + }
  70 +}
... ...
src/main/java/it/cnr/isti/hpc/trends/cli/ResiltechTrendsCLI.java
... ... @@ -19,6 +19,7 @@ import it.cnr.isti.hpc.cli.AbstractCommandLineInterface;
19 19 import it.cnr.isti.hpc.property.ProjectProperties;
20 20 import it.cnr.isti.hpc.trends.TrendDetector;
21 21 import it.cnr.isti.hpc.trends.listener.TwitterAPIListener;
  22 +import it.cnr.isti.hpc.trends.output.ConsoleCollector;
22 23 import it.cnr.isti.hpc.trends.output.FileTrendCollector;
23 24 import it.cnr.isti.hpc.trends.output.ResiltechCollector;
24 25  
... ... @@ -27,6 +28,8 @@ import javax.activity.InvalidActivityException;
27 28 import org.slf4j.Logger;
28 29 import org.slf4j.LoggerFactory;
29 30  
  31 +import ch.qos.logback.core.ConsoleAppender;
  32 +
30 33 /**
31 34 * @author Diego Ceccarelli <diego.ceccarelli@isti.cnr.it>
32 35 *
... ... @@ -52,10 +55,11 @@ public class ResiltechTrendsCLI extends AbstractCommandLineInterface {
52 55  
53 56 TrendDetector td = new TrendDetector().setMinFreq(minFreq)
54 57 .setBucketInterval(bucketInterval);
55   - td.addCollector(new FileTrendCollector("resiltech-trends.tsv")
56   - .asPlain());
57   - td.addCollector(new FileTrendCollector("resiltech-trends.json"));
58   - td.addCollector(new ResiltechCollector());
  58 + td.addCollector(new ConsoleCollector());
  59 + //td.addCollector(new FileTrendCollector("resiltech-trends.tsv")
  60 + //.asPlain());
  61 + //td.addCollector(new FileTrendCollector("resiltech-trends.json"));
  62 + //td.addCollector(new ResiltechCollector());
59 63 TwitterAPIListener listener = new TwitterAPIListener(td);
60 64 listener.run();
61 65 }
... ...
src/main/java/it/cnr/isti/hpc/trends/listener/TwitterAPIListener.java
... ... @@ -68,10 +68,12 @@ public class TwitterAPIListener extends AbstractTwitterListener {
68 68  
69 69 twitterStream.addListener(new StatusListener() {
70 70 public void onStatus(Status status) {
71   - String json = DataObjectFactory.getRawJSON(status);
  71 +
72 72 StatusJSONImpl s = new StatusJSONImpl();
  73 + s.setId(status.getId());
73 74 s.setTimestamp_ms(status.getCreatedAt().getTime());
74   - s.setText(status.getText());
  75 + s.setText(status.getText());
  76 + String json = DataObjectFactory.getRawJSON(status);
75 77 s.setJson(json);
76 78 collect(s);
77 79  
... ...
src/main/java/it/cnr/isti/hpc/trends/output/ConsoleCollector.java 0 โ†’ 100644
  1 +/**
  2 + * Copyright 2015 Diego Ceccarelli
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License");
  5 + * you may not use this file except in compliance with the License.
  6 + * You may obtain a copy of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS,
  12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 + * See the License for the specific language governing permissions and
  14 + * limitations under the License.
  15 + */
  16 +package it.cnr.isti.hpc.trends.output;
  17 +
  18 +import it.cnr.isti.hpc.io.IOUtils;
  19 +import it.cnr.isti.hpc.trends.util.StatusJSONImpl;
  20 +
  21 +import java.io.BufferedWriter;
  22 +import java.io.IOException;
  23 +import java.lang.reflect.Type;
  24 +
  25 +import twitter4j.Status;
  26 +
  27 +import com.google.gson.Gson;
  28 +import com.google.gson.GsonBuilder;
  29 +import com.google.gson.JsonDeserializationContext;
  30 +import com.google.gson.JsonDeserializer;
  31 +import com.google.gson.JsonElement;
  32 +import com.google.gson.JsonParseException;
  33 +import com.google.gson.JsonSerializationContext;
  34 +import com.google.gson.JsonSerializer;
  35 +
  36 +/**
  37 + * @author Diego Ceccarelli <diego.ceccarelli@isti.cnr.it>
  38 + *
  39 + * Created on Feb 7, 2015
  40 + */
  41 +public class ConsoleCollector extends TrendCollector {
  42 +
  43 +
  44 +
  45 + private static Gson gson = new GsonBuilder().registerTypeAdapter(
  46 + Status.class, new StatusSerializer()).create();
  47 +
  48 + public ConsoleCollector() {
  49 + }
  50 +
  51 +
  52 +
  53 + @Override
  54 + public void add(Trend t) throws IOException {
  55 + System.out.println("===================================");
  56 + System.out.println(t.getDescription());
  57 + for (Status s : t.getTweets()){
  58 + System.out.println("\t "+s.getText());
  59 + }
  60 + System.out.println("===================================");
  61 +
  62 + }
  63 +
  64 + @Override
  65 + public void close() throws IOException {
  66 +
  67 + }
  68 +
  69 + public static class StatusSerializer implements JsonSerializer<Status>,
  70 + JsonDeserializer<Status> {
  71 +
  72 + public JsonElement serialize(Status src, Type typeOfSrc,
  73 + JsonSerializationContext context) {
  74 + StatusJSONImpl i = (StatusJSONImpl) src;
  75 + return gson.fromJson(i.getJson(), JsonElement.class);
  76 + }
  77 +
  78 + public Status deserialize(JsonElement json, Type typeOfT,
  79 + JsonDeserializationContext context) throws JsonParseException {
  80 + // TODO Auto-generated method stub
  81 + return null;
  82 + }
  83 +
  84 + }
  85 +
  86 +}
... ...
src/main/java/it/cnr/isti/hpc/trends/output/FileTrendCollector.java
... ... @@ -63,6 +63,7 @@ public class FileTrendCollector extends TrendCollector {
63 63 String out = (format == Format.JSON) ? gson.toJson(t) : t.asString();
64 64 bw.write(out);
65 65 bw.newLine();
  66 + bw.flush();
66 67 }
67 68  
68 69 @Override
... ...