1th commit

This is a 1th commit for TwitterCrawler.
Gingber · Oct 16, 2013 · 1b17771 · 1b17771
1 parent e7205ec
commit 1b17771
Show file tree

Hide file tree

Showing 22 changed files with 927 additions and 0 deletions.
diff --git a/.classpath b/.classpath
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="src"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
+	<classpathentry kind="lib" path="lib/twitter4j-core-3.0.3.jar"/>
+	<classpathentry kind="lib" path="F:/workspace/TwitterCrawler/lib/mysql-connector-java-5.1.7-bin.jar"/>
+	<classpathentry kind="output" path="bin"/>
+</classpath>
diff --git a/.project b/.project
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>TwitterCrawler</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>
diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs
@@ -0,0 +1,3 @@
+eclipse.preferences.version=1
+encoding//src/twitter/crawler/OAuthTwitter.java=UTF-8
+encoding//src/twitter/crawler/StoreUser.java=UTF-8
diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,12 @@
+#Fri Sep 27 11:05:22 CST 2013
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
+org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
+org.eclipse.jdt.core.compiler.compliance=1.7
+org.eclipse.jdt.core.compiler.debug.lineNumber=generate
+org.eclipse.jdt.core.compiler.debug.localVariable=generate
+org.eclipse.jdt.core.compiler.debug.sourceFile=generate
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.source=1.7
diff --git a/bin/feedmonitor.properties b/bin/feedmonitor.properties
@@ -0,0 +1,3 @@
+feedurl=feed://yusuke.homeip.net/twitter4j/en/rss.xml
+id=YOUR_TWITTER_ID
+password=YOUR_TWITTER_PASSWORD
diff --git a/bin/hibernate.cfg.xml b/bin/hibernate.cfg.xml
@@ -0,0 +1,19 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<!DOCTYPE hibernate-configuration PUBLIC
+          "-//Hibernate/Hibernate Configuration DTD 3.0//EN"
+          "http://hibernate.sourceforge.net/hibernate-configuration-3.0.dtd">
+
+<!-- Generated by MyEclipse Hibernate Tools.                   -->
+<hibernate-configuration>
+
+    <session-factory>
+        <property name="dialect">org.hibernate.dialect.MySQLDialect</property>
+        <property name="connection.url">jdbc:mysql://127.0.0.1:3306/twitter</property>
+        <property name="connection.username">root</property>
+        <property name="connection.password">hadoop</property>
+        <property name="connection.driver_class">com.mysql.jdbc.Driver</property>
+        <property name="myeclipse.connection.profile">twitter</property>
+
+    </session-factory>
+
+</hibernate-configuration>
diff --git a/bin/oauth.properties b/bin/oauth.properties
@@ -0,0 +1,9 @@
+consumer.key = 1wNzFR8mlvcS3DFX10O3w
+consumer.secret = V9w1vWHUIHf0jm7LA65l4os9OvYvx9S9R4AEFUs
+request.token.key = 7PKsrEyU5rx4GtNk1BTKjfcmwNAimFm2ULiY9tJkHtQ
+request.token.secret = aUGFpz1F6mxKj6GALSjGJXFdljESD7DyBA4X6vTR53o
+access.token.key = 1588047948-Y1uss1iZafQYnBuaOVY3jzPoEBHCgIfst7JbFDw
+access.token.secret = 6XU1ehBD4BPFmJaGR0VHBcd1yqaaIsqwvQERZzA
+request.token.URL = https://twitter.com/oauth/request_token
+access.token.URL = https://twitter.com/oauth/access_token
+authorize.URL =https://twitter.com/oauth/authorize
diff --git a/bin/snscrawler.properties b/bin/snscrawler.properties
@@ -0,0 +1,9 @@
+consumer.key = 1wNzFR8mlvcS3DFX10O3w
+consumer.secret = V9w1vWHUIHf0jm7LA65l4os9OvYvx9S9R4AEFUs
+request.token.key = 7PKsrEyU5rx4GtNk1BTKjfcmwNAimFm2ULiY9tJkHtQ
+request.token.secret = aUGFpz1F6mxKj6GALSjGJXFdljESD7DyBA4X6vTR53o
+access.token.key = 1588047948-Y1uss1iZafQYnBuaOVY3jzPoEBHCgIfst7JbFDw
+access.token.secret = 6XU1ehBD4BPFmJaGR0VHBcd1yqaaIsqwvQERZzA
+request.token.URL = https://twitter.com/oauth/request_token
+access.token.URL = https://twitter.com/oauth/access_token
+authorize.URL =https://twitter.com/oauth/authorize
diff --git a/bin/twitter4j.properties b/bin/twitter4j.properties
@@ -0,0 +1,10 @@
+http.proxyHost = 127.0.0.1
+#http.proxyPort = 8580
+http.proxyPort = 9666
+#http.proxyPort = 8000
+#http.proxyPort = 8086
+#http.proxyPort = 8080
+#http.proxyPort = 1234
+#socks5.proxyHost=127.0.0.1
+#http.proxyPort=7070
+http.retryCount = 10
diff --git a/bin/xieyi64.properties b/bin/xieyi64.properties
@@ -0,0 +1,9 @@
+consumer.key = 6yPcng33vFYCimLH2V0g
+consumer.secret = Bo9MzDasz5Y4xZP1ctllhlvLQiBxvKpMLuTSC4k8oI
+request.token.key = B6vroPDgJ2KOxYxxrMGAjjM3HCAgFJx5CmBKK2HJJ0
+request.token.secret = UHqwL2dedZQLKXSXhwOZNL9anAGvtXsZ28KAbSU7gw
+access.token.key = 75471454-yeNtomMJcOU9mQfqRIbSS66970k5DbbXmzTChAcE8
+access.token.secret = wH7L4u8lnMCeQN2JhzeRVNZii3dnmEX25Z47wQIXg
+request.token.URL = http://twitter.com/oauth/request_token
+access.token.URL = http://twitter.com/oauth/access_token
+authorize.URL =http://twitter.com/oauth/authorize
diff --git a/src/feedmonitor.properties b/src/feedmonitor.properties
@@ -0,0 +1,3 @@
+feedurl=feed://yusuke.homeip.net/twitter4j/en/rss.xml
+id=YOUR_TWITTER_ID
+password=YOUR_TWITTER_PASSWORD
diff --git a/src/hibernate.cfg.xml b/src/hibernate.cfg.xml
@@ -0,0 +1,19 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<!DOCTYPE hibernate-configuration PUBLIC
+          "-//Hibernate/Hibernate Configuration DTD 3.0//EN"
+          "http://hibernate.sourceforge.net/hibernate-configuration-3.0.dtd">
+
+<!-- Generated by MyEclipse Hibernate Tools.                   -->
+<hibernate-configuration>
+
+    <session-factory>
+        <property name="dialect">org.hibernate.dialect.MySQLDialect</property>
+        <property name="connection.url">jdbc:mysql://127.0.0.1:3306/twitter</property>
+        <property name="connection.username">root</property>
+        <property name="connection.password">hadoop</property>
+        <property name="connection.driver_class">com.mysql.jdbc.Driver</property>
+        <property name="myeclipse.connection.profile">twitter</property>
+
+    </session-factory>
+
+</hibernate-configuration>
diff --git a/src/oauth.properties b/src/oauth.properties
@@ -0,0 +1,9 @@
+consumer.key = 1wNzFR8mlvcS3DFX10O3w
+consumer.secret = V9w1vWHUIHf0jm7LA65l4os9OvYvx9S9R4AEFUs
+request.token.key = 7PKsrEyU5rx4GtNk1BTKjfcmwNAimFm2ULiY9tJkHtQ
+request.token.secret = aUGFpz1F6mxKj6GALSjGJXFdljESD7DyBA4X6vTR53o
+access.token.key = 1588047948-Y1uss1iZafQYnBuaOVY3jzPoEBHCgIfst7JbFDw
+access.token.secret = 6XU1ehBD4BPFmJaGR0VHBcd1yqaaIsqwvQERZzA
+request.token.URL = https://twitter.com/oauth/request_token
+access.token.URL = https://twitter.com/oauth/access_token
+authorize.URL =https://twitter.com/oauth/authorize
diff --git a/src/snscrawler.properties b/src/snscrawler.properties
@@ -0,0 +1,9 @@
+consumer.key = 1wNzFR8mlvcS3DFX10O3w
+consumer.secret = V9w1vWHUIHf0jm7LA65l4os9OvYvx9S9R4AEFUs
+request.token.key = 7PKsrEyU5rx4GtNk1BTKjfcmwNAimFm2ULiY9tJkHtQ
+request.token.secret = aUGFpz1F6mxKj6GALSjGJXFdljESD7DyBA4X6vTR53o
+access.token.key = 1588047948-Y1uss1iZafQYnBuaOVY3jzPoEBHCgIfst7JbFDw
+access.token.secret = 6XU1ehBD4BPFmJaGR0VHBcd1yqaaIsqwvQERZzA
+request.token.URL = https://twitter.com/oauth/request_token
+access.token.URL = https://twitter.com/oauth/access_token
+authorize.URL =https://twitter.com/oauth/authorize
diff --git a/src/twitter/crawler/GetTwitterInfo.java b/src/twitter/crawler/GetTwitterInfo.java
@@ -0,0 +1,113 @@
+package twitter.crawler;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import twitter.crawler.util.DBUtil;
+import twitter4j.IDs;
+import twitter4j.RateLimitStatus;
+import twitter4j.RateLimitStatusListener;
+import twitter4j.Twitter;
+import twitter4j.TwitterException;
+import twitter4j.User;
+
+public class GetTwitterInfo {
+	private Twitter twitter;
+
+	public void getSpecifiedUser(String screenName){
+		OAuthTwitter oAuthTwitter=new OAuthTwitter();
+		Twitter twitter=oAuthTwitter.oAuthLogin();
+		try {
+			User user=twitter.showUser(screenName);
+			System.out.println(screenName+"'s URL:"+user.getURL());
+			System.out.println(screenName+"'s id:"+user.getId());
+			System.out.println(screenName+"'s Name:"+user.getName());
+			System.out.println(screenName+"'s CreateAt:"+user.getCreatedAt());
+			System.out.println(screenName+"'s Location:"+user.getLocation());
+			System.out.println(screenName+"'s Lang:"+user.getLang());
+			System.out.println(screenName+"'s Description:"+user.getDescription());
+			System.out.println(screenName+"'s StatusesCount:"+user.getStatusesCount());
+			System.out.println(screenName+"'s FollowersCount:"+user.getFollowersCount());
+			System.out.println(screenName+"'s FriendsCount:"+user.getFriendsCount());
+			System.out.println(screenName+"'s FavouritesCount:"+user.getFavouritesCount());
+			System.out.println(screenName+"'s CurrentStatus:"+user.getStatus().getText());
+		} catch (TwitterException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+	private void checkRateLimitStatus()  {
+		try {
+		RateLimitStatus limit = twitter.getRateLimitStatus().get("/application/rate_limit_status");
+		System.out.print("- limit: "+limit.getRemaining() +"\n");
+		if (limit.getRemaining() <= 2) {
+			int remainingTime = limit.getSecondsUntilReset() + 10;
+			System.out.println("Twitter request rate limit reached. Waiting "+remainingTime/60+" minutes to request again.");
+
+			try {
+				Thread.sleep(remainingTime*1000);
+			} catch (InterruptedException e) {
+				e.printStackTrace();
+			}
+		}
+		} catch (TwitterException te) {
+			System.err.println(te.getMessage());
+			if (te.getStatusCode()==503) {
+				try {
+					Thread.sleep(120*1000);// wait 2 minutes
+				} catch (InterruptedException e) {
+					e.printStackTrace();
+				} 
+			}
+		}
+		catch(Exception e) {
+			System.err.println(e.getMessage());
+
+		}
+	}
+	public void getRateLimitStatus() throws TwitterException{
+		OAuthTwitter oAuthTwitter=new OAuthTwitter();
+		twitter=oAuthTwitter.oAuthLogin();
+		RateLimitStatusListener listener=new RateLimitStatusListenerImpl();
+		twitter.addRateLimitStatusListener(listener);
+
+		try {
+			//twitter.showUser("xieyi64");
+			IDs users;
+			long cursor=-1;
+			Connection conn=DBUtil.getConn();
+			Statement stmt=DBUtil.createStmt(conn);
+			int j=0;
+			do{
+				//checkRateLimitStatus();
+				if (j==0)
+					users=twitter.getFollowersIDs("kaifulee",-1);
+				else
+					users=twitter.getFollowersIDs("kaifulee", cursor);
+
+				long[] ids=users.getIDs();
+				for (int i=0;i<ids.length;i++) {
+					String sql="insert into kaifulee(followerId,cursorStr) values("+ids[i]+",'"+cursor+"')";
+					DBUtil.update(sql, stmt);
+					System.out.println("update database success: "+(j*5000+i));
+				}
+				cursor=users.getNextCursor();
+				j++;
+
+				checkRateLimitStatus();
+
+			}while (users.hasNext());
+		} catch (TwitterException e) {
+			e.printStackTrace();
+		}finally{
+		}
+
+
+	}
+	public static void main(String[] args) throws TwitterException {
+		GetTwitterInfo getTwitterInfo=new GetTwitterInfo();
+		getTwitterInfo.getRateLimitStatus();
+		//getTwitterInfo.getSpecifiedUser("Barack Obama");
+	}
+}