View Javadoc

1   /**
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    */
4   package net.sourceforge.pmd.cpd;
5   
6   import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
7   import net.sourceforge.pmd.cpd.cppast.SimpleCharStream;
8   import net.sourceforge.pmd.cpd.cppast.Token;
9   import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
10  
11  import java.io.StringReader;
12  
13  public class CPPTokenizer implements Tokenizer {
14      private static SimpleCharStream charStream;
15  
16      public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
17          StringBuffer sb = sourceCode.getCodeBuffer();
18          try {
19              if (charStream == null) {
20                  charStream = new SimpleCharStream(new StringReader(sb.toString()));
21              } else {
22                  charStream.ReInit(new StringReader(sb.toString()));
23              }
24              CPPParserTokenManager.ReInit(charStream);
25              CPPParserTokenManager.setFileName(sourceCode.getFileName());
26              Token currToken = CPPParserTokenManager.getNextToken();
27              while (currToken.image.length() > 0) {
28                  tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine));
29                  currToken = CPPParserTokenManager.getNextToken();
30              }
31              tokenEntries.add(TokenEntry.getEOF());
32              System.err.println("Added " + sourceCode.getFileName());
33          } catch (TokenMgrError err) {
34              err.printStackTrace();
35              System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
36              tokenEntries.add(TokenEntry.getEOF());
37          }
38      }
39  }