View Javadoc
1   /**
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    */
4   package net.sourceforge.pmd.cpd;
5   
6   import java.io.BufferedReader;
7   import java.io.IOException;
8   import java.io.Reader;
9   import java.io.StringReader;
10  import java.util.Properties;
11  
12  import net.sourceforge.pmd.PMD;
13  import net.sourceforge.pmd.lang.LanguageRegistry;
14  import net.sourceforge.pmd.lang.LanguageVersionHandler;
15  import net.sourceforge.pmd.lang.TokenManager;
16  import net.sourceforge.pmd.lang.ast.TokenMgrError;
17  import net.sourceforge.pmd.lang.cpp.CppLanguageModule;
18  import net.sourceforge.pmd.lang.cpp.ast.Token;
19  import net.sourceforge.pmd.util.IOUtil;
20  
21  import org.apache.commons.io.IOUtils;
22  
23  /**
24   * The C++ tokenizer.
25   */
26  public class CPPTokenizer implements Tokenizer {
27  
28      private boolean skipBlocks = true;
29      private String skipBlocksStart;
30      private String skipBlocksEnd;
31  
32      /**
33       * Sets the possible options for the C++ tokenizer.
34       * @param properties the properties
35       * @see #OPTION_SKIP_BLOCKS
36       * @see #OPTION_SKIP_BLOCKS_PATTERN
37       */
38      public void setProperties(Properties properties) {
39          skipBlocks = Boolean.parseBoolean(properties.getProperty(OPTION_SKIP_BLOCKS, Boolean.TRUE.toString()));
40          if (skipBlocks) {
41              String skipBlocksPattern = properties.getProperty(OPTION_SKIP_BLOCKS_PATTERN, DEFAULT_SKIP_BLOCKS_PATTERN);
42              String[] split = skipBlocksPattern.split("\\|", 2);
43              skipBlocksStart = split[0];
44              if (split.length == 1) {
45                  skipBlocksEnd = split[0];
46              } else {
47                  skipBlocksEnd = split[1];
48              }
49          }
50      }
51  
52      @Override
53      public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
54          StringBuilder buffer = sourceCode.getCodeBuffer();
55          Reader reader = null;
56          try {
57              LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(CppLanguageModule.NAME)
58                      .getDefaultVersion().getLanguageVersionHandler();
59              reader = new StringReader(maybeSkipBlocks(buffer.toString()));
60              reader = IOUtil.skipBOM(reader);
61              TokenManager tokenManager = languageVersionHandler.getParser(
62                      languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
63              Token currentToken = (Token) tokenManager.getNextToken();
64              while (currentToken.image.length() > 0) {
65                  tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
66                  currentToken = (Token) tokenManager.getNextToken();
67              }
68              tokenEntries.add(TokenEntry.getEOF());
69              System.err.println("Added " + sourceCode.getFileName());
70          } catch (TokenMgrError err) {
71              err.printStackTrace();
72              System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
73              tokenEntries.add(TokenEntry.getEOF());
74          } catch (IOException e) {
75              e.printStackTrace();
76              System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
77              tokenEntries.add(TokenEntry.getEOF());
78          } finally {
79              IOUtils.closeQuietly(reader);
80          }
81      }
82  
83      private String maybeSkipBlocks(String test) throws IOException {
84          if (!skipBlocks) {
85              return test;
86          }
87  
88          BufferedReader reader = new BufferedReader(new StringReader(test));
89          StringBuilder filtered = new StringBuilder(test.length());
90          String line;
91          boolean skip = false;
92          while ((line = reader.readLine()) != null) {
93              if (skipBlocksStart.equalsIgnoreCase(line.trim())) {
94                  skip = true;
95              } else if (skip && skipBlocksEnd.equalsIgnoreCase(line.trim())) {
96                  skip = false;
97              }
98              if (!skip) {
99                  filtered.append(line);
100             }
101             filtered.append(PMD.EOL); // always add a new line to keep the line-numbering
102         }
103         return filtered.toString();
104     }
105 }