View Javadoc
1   /**
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    */
4   package net.sourceforge.pmd.cpd;
5   
6   import java.io.Reader;
7   import java.io.StringReader;
8   
9   import net.sourceforge.pmd.lang.LanguageRegistry;
10  import net.sourceforge.pmd.lang.LanguageVersionHandler;
11  import net.sourceforge.pmd.lang.TokenManager;
12  import net.sourceforge.pmd.lang.ast.TokenMgrError;
13  import net.sourceforge.pmd.lang.objectivec.ObjectiveCLanguageModule;
14  import net.sourceforge.pmd.lang.objectivec.ast.Token;
15  
16  import org.apache.commons.io.IOUtils;
17  
18  /**
19   * The Objective-C Tokenizer
20   */
21  public class ObjectiveCTokenizer implements Tokenizer {
22  
23      @Override
24      public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
25          StringBuilder buffer = sourceCode.getCodeBuffer();
26          Reader reader = null;
27          try {
28              LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(ObjectiveCLanguageModule.NAME)
29                      .getDefaultVersion().getLanguageVersionHandler();
30              reader = new StringReader(buffer.toString());
31              TokenManager tokenManager = languageVersionHandler.getParser(
32                      languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
33              Token currentToken = (Token) tokenManager.getNextToken();
34              while (currentToken.image.length() > 0) {
35                  tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
36                  currentToken = (Token) tokenManager.getNextToken();
37              }
38              tokenEntries.add(TokenEntry.getEOF());
39              System.err.println("Added " + sourceCode.getFileName());
40          } catch (TokenMgrError err) {
41              err.printStackTrace();
42              System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
43              tokenEntries.add(TokenEntry.getEOF());
44          } finally {
45              IOUtils.closeQuietly(reader);
46          }
47      }
48  }