Context Navigation

← Previous Revision
Latest Revision
Next Revision →
Annotate
Revision Log

Copy_2_of_Crawler.java @ 8056

Revision 8056, 10.6 KB checked in by gustavo, 11 years ago (diff)
Ticket #000 - Commit contendo o expresso com solr funcionando corretamente

Line
1	package pkg.crawler.data.conn;
2
3	import java.io.IOException;
4	import java.util.ArrayList;
5	import java.util.Date;
6	import java.util.List;
7	import java.util.Properties;
8
9	import javax.mail.Address;
10	import javax.mail.BodyPart;
11	import javax.mail.Folder;
12	import javax.mail.Message;
13	import javax.mail.MessagingException;
14	import javax.mail.Multipart;
15	import javax.mail.Part;
16	import javax.mail.Session;
17
18	import org.apache.solr.client.solrj.SolrServer;
19	import org.apache.solr.client.solrj.impl.HttpSolrServer;
20	import org.apache.solr.common.SolrInputDocument;
21
22	import pkg.crawler.entity.Message2SolrEntity;
23
24	import com.sun.mail.imap.ACL;
25	import com.sun.mail.imap.IMAPFolder;
26	import com.sun.mail.imap.IMAPMessage;
27	import com.sun.mail.imap.IMAPStore;
28	import com.sun.mail.imap.Rights;
29	import com.sun.mail.imap.Rights.Right;
30
31	public class Copy_2_of_Crawler{
32
33	private String host;
34	private String user;
35	private String password;
36	private String mbox;
37	private String hostSolr;
38	// private ArrayList<Message2SolrEntity> listIMAPMsgs;
39	private ACL acl;
40
41	public Copy_2_of_Crawler(String host, String password, String hostSolr)
42	{
43	this.host = host;
44	this.user = "expresso-admin";
45	this.password = password;
46	this.mbox = "user";
47	this.hostSolr = hostSolr;
48
49	//Cria a ACL a ser configurada
50	acl = new ACL("expresso-admin");
51
52	//Define direito de leitura
53	Rights rights = new Rights();
54	rights.add(Right.READ);
55
56	//Configura direito de leitura para a ACL
57	acl.setRights(rights);
58	}
59
60
61	public void run()
62	{
63
64	int iAux0 = 0;
65	while (iAux0 == 0)
66	{
67	iAux0 = 1;
68
69	//Inicializando as propriedades default da mÃ¡quina para conexÃ£o
70	//com o servidor IMAP
71	Properties props = System.getProperties();
72	Session session = Session.getInstance(props, null);
73
74	IMAPStore store = new IMAPStore(session, null);
75
76	try {
77
78	//Conectando com o servidor IMAP
79	store.connect(host, user, password);
80	//Definindo a pasta base a ser usada
81	IMAPFolder imapFolder = (IMAPFolder)store.getFolder(mbox);
82
83	//Faz a iteraÃ§Ã£o entre os usuÃ¡rios/pastas do usuÃ¡rio
84	for (Folder fINBOXAux : imapFolder.list())
85	{
86	System.out.println(fINBOXAux.getFullName());
87
88	//Cria uma instÃ¢ncia de conexÃ£o com o servidor Solr
89	SolrServer solrServer = new HttpSolrServer(hostSolr);
90
91	//Cria
92	List<Message2SolrEntity> listIMAPMsgs = new ArrayList<Message2SolrEntity>();
93	//INBOX
94	IMAPFolder imapFINBOXAux = (IMAPFolder)fINBOXAux;
95
96	//Adiciona ACL de permissÃ£o de leitura, para o usuÃ¡rio expresso-admin conseguir fazer a leitura
97	try
98	{
99	imapFINBOXAux.addACL(acl);
100	}
101	catch (Exception e)
102	{
103	continue;
104	}
105
106	//Abre a caixa de e-mail
107	imapFINBOXAux.open(Folder.READ_ONLY);
108
109	//Carrega as mensagens da caixa para um array
110	Message[] msgsINBOX = imapFINBOXAux.getMessages();
111
112	//Faz a iteraÃ§Ã£o entre as mensagens
113	for (Message msgAuxINBOX : msgsINBOX)
114	{
115
116	IMAPMessage m = (IMAPMessage)msgAuxINBOX;
117
118	//adiciona as mensagens na varÃavel listIMAPMsgs
119	try
120	{
121	dumpPart(m, listIMAPMsgs);
122	}catch(Exception e)
123	{
124	continue;
125	}
126	}
127	imapFINBOXAux.removeACL("expresso-admin");
128	for(int i = 0; i < listIMAPMsgs.size(); i++ )
129	{
130	SolrInputDocument doc = new SolrInputDocument();
131	doc.addField("id", listIMAPMsgs.get(i).getId().toString());
132	doc.addField("user", listIMAPMsgs.get(i).getUser().toString());
133	doc.addField("folder", listIMAPMsgs.get(i).getFolder().toString());
134	doc.addField("msg_no", listIMAPMsgs.get(i).getMsgNo());
135	doc.addField("from", listIMAPMsgs.get(i).getFrom().toString());
136	doc.addField("to", listIMAPMsgs.get(i).getTo().toString());
137	doc.addField("subject", listIMAPMsgs.get(i).getSubject().toString());
138	doc.addField("content", listIMAPMsgs.get(i).getContent().toString());
139	doc.addField("copyto", listIMAPMsgs.get(i).getCopyto().toString());
140	doc.addField("sent_date", listIMAPMsgs.get(i).getSent_date());
141	doc.addField("hiddencopyto", listIMAPMsgs.get(i).getHiddencopyto().toString());
142
143	try {
144	solrServer.add(doc);
145	} catch (Exception e) {
146	System.err.println("solr -> " + e.getMessage());
147	}
148
149	}
150
151	listIMAPMsgs = null;
152
153	solrServer.commit();
154
155	solrServer = null;
156
157	for (Folder fAux : imapFINBOXAux.list())
158	{
159	//Verifica se nÃ£o Ã© uma pasta compartilhada
160	if(fAux.getFullName().split("/").length <= 3 && !fAux.getFullName().split("/")[2].equals("user"))
161	{
162	//INBOX
163	IMAPFolder imapFAux = (IMAPFolder)fAux;
164
165	crawIntoUserFolders(imapFAux);
166
167	}
168	}
169	Thread.sleep(30000);
170
171	System.out.println(imapFINBOXAux.getFullName());
172	}
173	store.close();
174
175
176
177	} catch (MessagingException e) {
178	System.out.println(e.getMessage());
179	e.printStackTrace();
180	}
181	catch (Exception e)
182	{
183	System.out.println(e.getMessage());
184	e.printStackTrace();
185	}
186	}
187	}
188
189	private void crawIntoUserFolders(IMAPFolder imapFAux) throws Exception
190	{
191	SolrServer solrServer = new HttpSolrServer(hostSolr);
192
193	List<Message2SolrEntity> listIMAPMsgs = new ArrayList<Message2SolrEntity>();
194
195	try
196	{
197	imapFAux.addACL(acl);
198	}catch(Exception e)
199	{
200	return;
201	}
202
203	imapFAux.open(Folder.READ_ONLY);
204
205	Message[] msgs= imapFAux.getMessages();
206
207	for (Message msgAux: msgs)
208	{
209	IMAPMessage m = (IMAPMessage)msgAux;
210	try
211	{
212	dumpPart(m, listIMAPMsgs);
213	}catch(Exception e)
214	{
215	continue;
216	}
217	}
218	imapFAux.removeACL("expresso-admin");
219	System.out.println(imapFAux.getFullName());
220	for(int i = 0; i < listIMAPMsgs.size(); i++ )
221	{
222	SolrInputDocument doc = new SolrInputDocument();
223	doc.addField("id", listIMAPMsgs.get(i).getId());
224	doc.addField("user", listIMAPMsgs.get(i).getUser());
225	doc.addField("folder", listIMAPMsgs.get(i).getFolder());
226	doc.addField("msg_no", listIMAPMsgs.get(i).getMsgNo());
227	doc.addField("from", listIMAPMsgs.get(i).getFrom());
228	doc.addField("to", listIMAPMsgs.get(i).getTo());
229	doc.addField("subject", listIMAPMsgs.get(i).getSubject());
230	doc.addField("content", listIMAPMsgs.get(i).getContent());
231	doc.addField("copyto", listIMAPMsgs.get(i).getCopyto());
232	doc.addField("sent_date", listIMAPMsgs.get(i).getSent_date());
233	doc.addField("hiddencopyto", listIMAPMsgs.get(i).getHiddencopyto());
234
235	}
236	listIMAPMsgs = null;
237	solrServer.commit();
238	solrServer = null;
239	}
240
241	public void dumpPart(IMAPMessage m, List<Message2SolrEntity> listIMAPMsgs) throws Exception
242	{
243	//Verifica se possui identificador para poder continuar
244	if(m.getMessageID() == null \|\| m.getMessageID().equals("")){
245	return;
246	}
247
248	Message2SolrEntity msgEntity = null;
249	msgEntity = new Message2SolrEntity();
250
251	msgEntity.setId(new StringBuilder( m.getMessageID()));
252
253	String user = m.getFolder().getFullName().split("/")[1];
254	msgEntity.setUser(new StringBuilder(user));
255
256	String folder = m.getFolder().getFullName().split("/")
257	[m.getFolder().getFullName().split("/").length-1];
258
259	if(!user.trim().equals(folder.trim()))
260	{
261	msgEntity.setFolder(new StringBuilder(folder));
262	}
263	else
264	{
265	msgEntity.setFolder(new StringBuilder("INBOX"));
266	}
267
268	msgEntity.setMsgNo(new StringBuilder(String.valueOf(m.getMessageNumber())));
269
270
271	if(m.getSubject() != null){
272	msgEntity.setSubject(new StringBuilder( m.getSubject() ));
273	}
274	Address[] a;
275	// FROM
276	if ((a = m.getFrom()) != null) {
277	for (int j = 0; j < a.length; j++)
278	{
279	if(msgEntity.getFrom() == null)
280	{
281	msgEntity.setFrom(new StringBuilder(a[j].toString()));
282	}
283	else
284	{
285	msgEntity.setFrom(msgEntity.getFrom().append(", ").append(a[j].toString()));
286	}
287	// System.out.println("FROM: " + a[j].toString());
288	}
289
290	}
291
292	// TO
293	if ((a = m.getRecipients(Message.RecipientType.TO)) != null) {
294	for (int j = 0; j < a.length; j++)
295	{
296	if(msgEntity.getTo() == null)
297	{
298	msgEntity.setTo(new StringBuilder(a[j].toString()));
299	}
300	else
301	{
302	msgEntity.setTo(msgEntity.getTo().append(", ").append(a[j].toString()));
303	}
304	}
305	}
306
307	// CC
308	if ((a = m.getRecipients(Message.RecipientType.CC)) != null) {
309	for (int j = 0; j < a.length; j++)
310	{
311	if(msgEntity.getCopyto() == null)
312	{
313	msgEntity.setCopyto(new StringBuilder(a[j].toString()));
314	}
315	else
316	{
317	msgEntity.setCopyto(msgEntity.getCopyto().append(", ").append(a[j].toString()));
318	}
319	}
320	}
321	// CC
322	if ((a = m.getRecipients(Message.RecipientType.BCC)) != null) {
323	for (int j = 0; j < a.length; j++)
324	{
325	if(msgEntity.getHiddencopyto() == null)
326	{
327	msgEntity.setHiddencopyto(new StringBuilder(a[j].toString()));
328	}
329	else
330	{
331	msgEntity.setHiddencopyto(new StringBuilder(msgEntity.getHiddencopyto() + ","+a[j].toString()));
332	}
333	}
334	}
335
336	// DATE
337	Date d = m.getSentDate();
338	if(msgEntity != null)
339	{
340	if(d != null)
341	{
342	msgEntity.setSent_date(new StringBuilder(String.valueOf(d.getTime())));
343	}
344	else
345	{
346	msgEntity.setSent_date(new StringBuilder(String.valueOf(new Date().getTime())));
347	}
348	}
349
350	try{
351	Object o = m.getContent();
352	if (o instanceof String) {
353	if( msgEntity != null )
354	{
355	msgEntity.setContent(new StringBuilder( o.toString()));
356	}
357	} else if (o instanceof Multipart) {
358	Multipart multipart = (Multipart)o;
359	int count = multipart.getCount();
360
361	for (int i = 0; i < count; i++)
362	{
363	if(msgEntity.getContent() == null \|\| msgEntity.getContent().toString().trim().equals(""))
364	{
365	msgEntity.setContent(new StringBuilder(""));
366	}
367	if(multipart.getBodyPart(i).getContentType() != Part.ATTACHMENT)
368	{
369	msgEntity.setContent(msgEntity.getContent().append(" ").append(getPlainContent(multipart.getBodyPart(i))));
370	}
371
372	}
373
374	}
375	}catch (Exception e){
376	System.out.println("Exception ocurred!");
377	e.printStackTrace();
378	}
379
380	if(msgEntity != null && msgEntity.getId() != null){
381	listIMAPMsgs.add(msgEntity);
382	}
383
384	}
385
386	private StringBuilder getPlainContent(BodyPart bodyPart) throws IOException, MessagingException
387	{
388	try
389	{
390
391	if(bodyPart.getContent() instanceof String)
392	{
393	// ByteArrayOutputStream out = new ByteArrayOutputStream();
394	// InputStream in = bodyPart.getInputStream();
395	//
396	// byte[] buffer = new byte[1024];
397	// byte[] data = null;
398	//
399	// while(in.read(buffer) != -1) {
400	// out.write(buffer);
401	// }
402	//
403	// data = out.toByteArray();
404
405	StringBuilder text = new StringBuilder(bodyPart.getContent().toString());
406
407	// out.close();
408	// in.close();
409
410	return text;
411	}
412	}catch(Exception e)
413	{
414	System.out.println(e.getMessage());
415	e.printStackTrace();
416	return new StringBuilder("");
417	}
418
419
420	return new StringBuilder("");
421	}
422
423	}

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: sandbox/expresso-solr/imapCrawler/SRC/JCrawlerForIMAP (cópia funcional)/src/pkg/crawler/data/conn/Copy_2_of_Crawler.java @ 8056

Download in other formats: