View Javadoc

1   /**
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    */
4   package net.sourceforge.pmd.cpd;
5   
6   import static org.junit.Assert.assertEquals;
7   import net.sourceforge.pmd.PMD;
8   
9   import org.junit.Test;
10  
11  public class JavaTokensTokenizerTest {
12  
13      @Test
14      public void test1() throws Throwable {
15          Tokenizer tokenizer = new JavaTokenizer();
16          SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("public class Foo {}"));
17          Tokens tokens = new Tokens();
18          tokenizer.tokenize(sourceCode, tokens);
19          assertEquals(6, tokens.size());
20          assertEquals("public class Foo {}", sourceCode.getSlice(1, 1));
21      }
22  
23      @Test
24      public void testCommentsIgnored() throws Throwable {
25          Tokenizer tokenizer = new JavaTokenizer();
26          SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("public class Foo { // class Bar */ \n }"));
27          Tokens tokens = new Tokens();
28          tokenizer.tokenize(sourceCode, tokens);
29          assertEquals(6, tokens.size());
30      }
31  
32      @Test
33      public void test2() throws Throwable {
34          Tokenizer t = new JavaTokenizer();
35          String data = "public class Foo {" + PMD.EOL + "public void bar() {}" + PMD.EOL + "public void buz() {}" + PMD.EOL + "}";
36          SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader(data));
37          Tokens tokens = new Tokens();
38          t.tokenize(sourceCode, tokens);
39          assertEquals("public class Foo {" + PMD.EOL + "public void bar() {}", sourceCode.getSlice(1, 2));
40      }
41  
42      @Test
43      public void testDiscardSemicolons() throws Throwable {
44          Tokenizer t = new JavaTokenizer();
45          SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("public class Foo {private int x;}"));
46          Tokens tokens = new Tokens();
47          t.tokenize(sourceCode, tokens);
48          assertEquals(9, tokens.size());
49      }
50  
51      @Test
52      public void testDiscardImports() throws Throwable {
53          Tokenizer t = new JavaTokenizer();
54          SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("import java.io.File;" + PMD.EOL + "public class Foo {}"));
55          Tokens tokens = new Tokens();
56          t.tokenize(sourceCode, tokens);
57          assertEquals(6, tokens.size());
58      }
59  
60      @Test
61      public void testDiscardPkgStmts() throws Throwable {
62          Tokenizer t = new JavaTokenizer();
63          SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("package foo.bar.baz;" + PMD.EOL + "public class Foo {}"));
64          Tokens tokens = new Tokens();
65          t.tokenize(sourceCode, tokens);
66          assertEquals(6, tokens.size());
67      }
68  
69      @Test
70      public void testDiscardSimpleOneLineAnnotation() throws Throwable {
71          JavaTokenizer t = new JavaTokenizer();
72          t.setIgnoreAnnotations(true);
73          SourceCode sourceCode = new SourceCode(
74                  new SourceCode.StringCodeLoader(
75                      "package foo.bar.baz;" +
76                      PMD.EOL +
77                      "@MyAnnotation" +
78                      PMD.EOL +
79                      "public class Foo {}"
80                  ));
81          Tokens tokens = new Tokens();
82          t.tokenize(sourceCode, tokens);
83          assertEquals(6, tokens.size());
84      }
85  
86      /**
87       * Comments are discarded already by the Java parser.
88       * It would be nice, however, to use simple comments like
89       * //CPD-START or //CPD-END
90       * to enable discard-mode of CPD
91       */
92      @Test
93      public void testIgnoreComments() {
94          JavaTokenizer t = new JavaTokenizer();
95          t.setIgnoreAnnotations(false);
96          SourceCode sourceCode = new SourceCode(
97                  new SourceCode.StringCodeLoader(
98                      "package foo.bar.baz;" +
99                      PMD.EOL +
100                     "/*****" +
101                     PMD.EOL +
102                     " * ugh" +
103                     PMD.EOL +
104                     " *****/" +
105                     PMD.EOL +
106                     "public class Foo {}"
107                 ));
108         Tokens tokens = new Tokens();
109         t.tokenize(sourceCode, tokens);
110         assertEquals(6, tokens.size());
111     }
112 
113     @Test
114     public void testDiscardOneLineAnnotationWithParams() throws Throwable {
115         JavaTokenizer t = new JavaTokenizer();
116         t.setIgnoreAnnotations(true);
117 
118         SourceCode sourceCode = new SourceCode(
119                 new SourceCode.StringCodeLoader(
120                     "package foo.bar.baz;" +
121                     PMD.EOL +
122                     "@ MyAnnotation (\"ugh\")" +
123                     PMD.EOL +
124                     "@NamedQueries({" +
125                             PMD.EOL +
126                     "@NamedQuery(" +
127                             PMD.EOL +
128                     ")})" +
129                             PMD.EOL +
130                     "public class Foo {" +
131                             PMD.EOL +
132                             "}"
133                 ));
134         Tokens tokens = new Tokens();
135         t.tokenize(sourceCode, tokens);
136         TokenEntry.getEOF();
137         assertEquals(6, tokens.size());
138     }
139 
140     @Test
141     public void testIgnoreBetweenSpecialAnnotation() throws Throwable {
142         JavaTokenizer t = new JavaTokenizer();
143         t.setIgnoreAnnotations(false);
144         SourceCode sourceCode = new SourceCode(
145                 new SourceCode.StringCodeLoader(
146                     "package foo.bar.baz;" +
147                     PMD.EOL +
148                     "@SuppressWarnings({\"woof\",\"CPD-START\"})" +
149                     PMD.EOL +
150                     "@SuppressWarnings(\"CPD-START\")" +
151                     PMD.EOL +
152 
153                     "@ MyAnnotation (\"ugh\")" +
154                     PMD.EOL +
155                     "@NamedQueries({" +
156                             PMD.EOL +
157                     "@NamedQuery(" +
158                             PMD.EOL +
159                     ")})" +
160                             PMD.EOL +
161                     "public class Foo {}" +
162                     "@SuppressWarnings({\"ugh\",\"CPD-END\"})" +
163                     PMD.EOL
164 
165                 ));
166         Tokens tokens = new Tokens();
167         t.tokenize(sourceCode, tokens);
168         TokenEntry.getEOF();
169         assertEquals(10, tokens.size());
170     }
171 
172 
173     @Test
174     public void testIgnoreBetweenSpecialAnnotationAndIgnoreAnnotations() throws Throwable {
175         JavaTokenizer t = new JavaTokenizer();
176         t.setIgnoreAnnotations(true);
177         SourceCode sourceCode = new SourceCode(
178                 new SourceCode.StringCodeLoader(
179                     "package foo.bar.baz;" +
180                     PMD.EOL +
181                     "@SuppressWarnings({\"woof\",\"CPD-START\"})" +
182                     PMD.EOL +
183                     "@SuppressWarnings(\"CPD-START\")" +
184                     PMD.EOL +
185 
186                     "@ MyAnnotation (\"ugh\")" +
187                     PMD.EOL +
188                     "@NamedQueries({" +
189                             PMD.EOL +
190                     "@NamedQuery(" +
191                             PMD.EOL +
192                     ")})" +
193                             PMD.EOL +
194                     "public class Foo {}" +
195                             PMD.EOL +
196                     "@SuppressWarnings({\"ugh\",\"CPD-END\"})" +
197                     PMD.EOL
198 
199                 ));
200         Tokens tokens = new Tokens();
201         t.tokenize(sourceCode, tokens);
202         TokenEntry.getEOF();
203         assertEquals(1, tokens.size());
204     }
205 
206 
207     public static junit.framework.Test suite() {
208         return new junit.framework.JUnit4TestAdapter(JavaTokensTokenizerTest.class);
209     }
210 }
211 
212