Merge branch 'master' of https://github.com/diegoceccarelli/json-wikipedia

Diego Ceccarelli
2 parents 20bcb44a 986670ad
Showing 4 changed files with 43 additions and 13 deletions Show diff stats
CREDITS.txt
README.md
src/main/java/it/cnr/isti/hpc/wikipedia/article/Article.java
src/main/java/it/cnr/isti/hpc/wikipedia/parser/ArticleParser.java
+## CREDITS.txt
+
+
+### Contributors
+
+Mike Huffman - added the external links for each article;
+Sara Montesi - http://www.saramontesi.it/ - json-wikipedia logo;
 \ No newline at end of file
 json-wikipedia ![json-wikipedia](https://dl.dropboxusercontent.com/u/4663256/tmp/json-wikipedia.png) 
 ==============
  
- Json Wikipedia contains code to convert the Wikipedia XML dump in a [JSON][json] dump.
+ Json Wikipedia contains code to convert the Wikipedia XML dump into a [JSON][json] dump.
  
 #### Setup ####
  
@@ -19,7 +19,7 @@ or
  
 	./scripts/convert-xml-dump-to-json.sh [en|it] wikipedia-dump.xml.bz wikipedia-dump.json[.gz]
  
-produces in `wikipedia-dump.json` the JSON version of the dump. Each line of the file contains an article 
+produces in `wikipedia-dump.json` the JSON version of the dump ([here you can find an example](https://dl.dropboxusercontent.com/u/4663256/tmp/json-wikipedia-sample.json)). Each line of the file contains an article 
 of dump encoded in JSON. Each JSON line can be deserialized in an [Article](http://sassicaia.isti.cnr.it/javadocs/json-wikipedia/it/cnr/isti/hpc/wikipedia/article/Article.html) object, 
 which represents an 
 _enriched_ version of the wikitext page. The Article object contains: 
@@ -34,13 +34,14 @@ _enriched_ version of the wikitext page. The Article object contains:
   * a list of  tables that appear in the article ;
   * a list of lists that  that appear in the article ;
   * a list  of internal links that appear in the article;
+  * a list of external links that appear in the article;
   * if the article  is a redirect, the pointed article;
   * a list of section titles in the article;
-  * the text of the article, divided in paragraphs;
+  * the text of the article, divided in paragraphs (PLAIN, no wikitext);
   * the categories and the templates of the articles;
   * the list of attributes found in the templates;
   * a list of terms highlighted in the article;
-  * if present, the infobox.
+  * if present, the infobox. 
  
 #### Usage ####
  
@@ -4,7 +4,7 @@
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
@@ -23,7 +23,7 @@ import com.google.gson.Gson;
  
 /**
  * Article represents an article in the Wikipedia dump.
- * 
+ *
  * @author Diego Ceccarelli, diego.ceccarelli@isti.cnr.it created on 19/nov/2011
  */
 public class Article {
@@ -57,6 +57,7 @@ public class Article {
 	private List<Link> images;
 	protected List<List<String>> lists;
 	private List<Link> links;
+	private List<Link> externalLinks;
 	protected String redirect;
 	private List<String> sections;
 	private List<String> paragraphs;
@@ -131,7 +132,7 @@ public class Article {
  
 	/**
 	 * the redirect without the anchor, e.g., da_vinci#life -> da_vinci
-	 * 
+	 *
 	 * @return the redirect without the anchor
 	 */
 	public String getRedirectNoAnchor() {
@@ -386,6 +387,10 @@ public class Article {
 		for (Link l : getLinks())
 			sb.append("\t").append(l).append("\n");
  
+		sb.append("EXTERNALLINKS:\n");
+		for (Link l : getExternalLinks())
+			sb.append("\t").append(l).append("\n");
+
 		sb.append("CATEGORIES:\n");
 		for (Link l : getCategories())
 			sb.append("\t").append(l).append("\n");
@@ -397,7 +402,7 @@ public class Article {
  
 	/**
 	 * Removes the TEMPLATE text from the row text of the article.
-	 * 
+	 *
 	 * @param text
 	 * @return the 'cleaned' text
 	 */
@@ -451,10 +456,20 @@ public class Article {
 		return links;
 	}
  
+	public List<Link> getExternalLinks() {
+		if (externalLinks == null)
+			return Collections.emptyList();
+		return externalLinks;
+	}
+
 	public void setLinks(List<Link> links) {
 		this.links = links;
 	}
  
+	public void setExternalLinks(List<Link> links) {
+		this.externalLinks = links;
+	}
+
 	public void addCategory(Link category) {
 		if (this.getCategories() == null)
 			categories = new ArrayList<Link>();
@@ -4,7 +4,7 @@
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
  *  You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  *  Unless required by applicable law or agreed to in writing, software
@@ -44,11 +44,11 @@ import de.tudarmstadt.ukp.wikipedia.parser.mediawiki.MediaWikiParser;
 /**
  * Generates a Mediawiki parser given a language, (it will expect to find a
  * locale file in <tt>src/main/resources/</tt>).
- * 
+ *
  * @see Locale
- * 
+ *
  * @author Diego Ceccarelli <diego.ceccarelli@isti.cnr.it>
- * 
+ *
  *         Created on Feb 14, 2013
  */
 public class ArticleParser {
@@ -325,6 +325,7 @@ public class ArticleParser {
 	private void setLinks(Article article, ParsedPage page) {
  
 		List<Link> links = new ArrayList<Link>(10);
+		List<Link> elinks = new ArrayList<Link>(10);
  
 		for (de.tudarmstadt.ukp.wikipedia.parser.Link t : page.getLinks()) {
 			if (t.getType() == de.tudarmstadt.ukp.wikipedia.parser.Link.type.INTERNAL) {
@@ -332,8 +333,14 @@ public class ArticleParser {
 				links.add(new Link(t.getTarget(), t.getText()));
  
 			}
+			if (t.getType() == de.tudarmstadt.ukp.wikipedia.parser.Link.type.EXTERNAL) {
+
+				elinks.add(new Link(t.getTarget(), t.getText()));
+
+			}
 		}
 		article.setLinks(links);
+		article.setExternalLinks(elinks);
 	}
  
 	private void setTemplates(Article article, ParsedPage page) {
@@ -355,7 +362,7 @@ public class ArticleParser {
 	}
  
 	/**
-	 * 
+	 *
 	 * @param templateParameters
 	 */
 	private void parseTemplatesSchema(Article article,