View Javadoc

1   /**
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    */
4   package net.sourceforge.pmd.cpd;
5   
6   import java.io.StringReader;
7   
8   import net.sourceforge.pmd.lang.LanguageVersion;
9   import net.sourceforge.pmd.lang.LanguageVersionHandler;
10  import net.sourceforge.pmd.lang.TokenManager;
11  import net.sourceforge.pmd.lang.ast.TokenMgrError;
12  import net.sourceforge.pmd.lang.cpp.ast.Token;
13  
14  import org.apache.commons.io.IOUtils;
15  
16  public class CPPTokenizer implements Tokenizer {
17  
18  	public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
19  		StringBuilder buffer = sourceCode.getCodeBuffer();
20  		StringReader reader = null;
21  		try {
22  			LanguageVersionHandler languageVersionHandler = LanguageVersion.CPP.getLanguageVersionHandler();
23  			reader = new StringReader(buffer.toString());
24  			TokenManager tokenManager = languageVersionHandler.getParser(
25  					languageVersionHandler.getDefaultParserOptions())
26  					.getTokenManager(sourceCode.getFileName(), reader);
27  			Token currentToken = (Token) tokenManager.getNextToken();
28  			while (currentToken.image.length() > 0) {
29  				tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
30  				currentToken = (Token) tokenManager.getNextToken();
31  			}
32  			tokenEntries.add(TokenEntry.getEOF());
33  			System.err.println("Added " + sourceCode.getFileName());
34  		} catch (TokenMgrError err) {
35  			err.printStackTrace();
36  			System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
37  			tokenEntries.add(TokenEntry.getEOF());
38  		} finally {
39  			IOUtils.closeQuietly(reader);
40  		}
41  	}
42  }