Attachment #36564 for bug #89000

This Bugzilla instance is a read-only archive of historic NetBeans bug reports. To report a bug in NetBeans please follow the project's instructions for reporting issues.

View | Details | Raw Unified | Return to bug 89000
Collapse All | Expand All

Lines 719-722 Link Here

(-)html/lexer/src/org/netbeans/lib/html/lexer/HTMLLexer.java (+3 lines)
719	return tokenFactory.createToken(tokenId);	719	return tokenFactory.createToken(tokenId);
720	}	720	}
721		721
		722	public void release() {
		723	}
		724
722	}	725	}

Lines 933-936 Link Here

(-)java/lexer/src/org/netbeans/lib/java/lexer/JavaLexer.java (+3 lines)
933	: tokenFactory.createToken(id);	933	: tokenFactory.createToken(id);
934	}	934	}
935		935
		936	public void release() {
		937	}
		938
936	}	939	}

Lines 105-108 Link Here

(-)java/lexer/src/org/netbeans/lib/java/lexer/JavaStringLexer.java (+3 lines)
105	return tokenFactory.createToken(id);	105	return tokenFactory.createToken(id);
106	}	106	}
107		107
		108	public void release() {
		109	}
		110
108	}	111	}

Lines 111-114 Link Here

(-)java/lexer/src/org/netbeans/lib/java/lexer/JavadocLexer.java (+3 lines)
111	return tokenFactory.createToken(id);	111	return tokenFactory.createToken(id);
112	}	112	}
113		113
		114	public void release() {
		115	}
		116
114	}	117	}

Lines 1-4 Link Here

(-)lexer/manifest.mf (-1 / +1 lines)
1	OpenIDE-Module: org.netbeans.modules.lexer/2	1	OpenIDE-Module: org.netbeans.modules.lexer/2
2	OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/lexer/Bundle.properties	2	OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/lexer/Bundle.properties
3	OpenIDE-Module-Specification-Version: 1.11.0	3	OpenIDE-Module-Specification-Version: 1.12.0
4	OpenIDE-Module-Recommends: org.netbeans.spi.lexer.LanguageProvider	4	OpenIDE-Module-Recommends: org.netbeans.spi.lexer.LanguageProvider




<!-- ACTUAL CHANGES BEGIN HERE: -->

  <changes>
  <change id="api.lexer.release">
      <api name="api"/>
      <summary>Added Lexer.release()</summary>
      <version major="1" minor="12"/>
      <date day="7" month="12" year="2006"/>
      <author login="mmetelka"/>
      <compatibility source="incompatible" deletion="no" addition="yes" modification="no"/>
      <description>
        <p>
            Added <code>Lexer.release()</code> useful for lexer instances caching.
        </p>
      </description>
  </change>

  <change id="api.customembedding">
      <api name="api"/>
      <summary>Adding custom embedding creation TokenSequence.createEmbedding()</summary>

Lines 181-184 Link Here

(-)lexer/editorbridge/calc/src/org/netbeans/modules/lexer/editorbridge/calc/lang/CalcLexer.java (+3 lines)
181	: tokenFactory.createToken(id);	181	: tokenFactory.createToken(id);
182	}	182	}
183		183
		184	public void release() {
		185	}
		186
184	}	187	}

Lines 50-53 Link Here

(-)lexer/nbbridge/test/unit/src/org/netbeans/modules/lexer/nbbridge/test/simple/SimpleCharLexer.java (+3 lines)
50	return null; // always in default state after token recognition	50	return null; // always in default state after token recognition
51	}	51	}
52		52
		53	public void release() {
		54	}
		55
53	}	56	}

Lines 96-99 Link Here

(-)lexer/nbbridge/test/unit/src/org/netbeans/modules/lexer/nbbridge/test/simple/SimplePlainLexer.java (+3 lines)
96	return null; // always in default state after token recognition	96	return null; // always in default state after token recognition
97	}	97	}
98		98
		99	public void release() {
		100	}
		101
99	}	102	}

Lines 115-120 Link Here

(-)lexer/src/org/netbeans/lib/lexer/EmbeddedTokenList.java (+2 lines)
115	}	115	}
116	token = lexerInputOperation.nextToken();	116	token = lexerInputOperation.nextToken();
117	}	117	}
		118	lexerInputOperation.release();
		119	lexerInputOperation = null;
118		120
119	trimToSize(); // Compact storage	121	trimToSize(); // Compact storage
120	if (laState != null)	122	if (laState != null)

Lines 366-369 Link Here

(-)lexer/src/org/netbeans/lib/lexer/LexerInputOperation.java (+4 lines)
366	return tokenList.inputAttributes();	366	return tokenList.inputAttributes();
367	}	367	}
368		368
		369	public final void release() {
		370	lexer.release();
		371	}
		372
369	}	373	}

Lines 168-173 Link Here

(-)lexer/src/org/netbeans/lib/lexer/batch/BatchTokenList.java (+1 lines)
168	lexerInputOperation.lexerState());	168	lexerInputOperation.lexerState());
169	}	169	}
170	} else { // no more tokens from lexer	170	} else { // no more tokens from lexer
		171	lexerInputOperation.release();
171	lexerInputOperation = null;	172	lexerInputOperation = null;
172	trimToSize();	173	trimToSize();
173	}	174	}

Lines 154-159 Link Here

(-)lexer/src/org/netbeans/lib/lexer/inc/IncTokenList.java (+4 lines)
154	laState = laState.add(lexerInputOperation.lookahead(),	154	laState = laState.add(lexerInputOperation.lookahead(),
155	lexerInputOperation.lexerState());	155	lexerInputOperation.lexerState());
156	} else { // no more tokens from lexer	156	} else { // no more tokens from lexer
		157	lexerInputOperation.release();
157	lexerInputOperation = null;	158	lexerInputOperation = null;
158	trimToSize();	159	trimToSize();
159	laState.trimToSize();	160	laState.trimToSize();
Lines 281-286 Link Here
281	}	282	}
282		283
283	public void refreshLexerInputOperation() {	284	public void refreshLexerInputOperation() {
		285	// Only called when !isFullyLexed() but "inited" might be false => must check "!= null"
		286	if (lexerInputOperation != null)
		287	lexerInputOperation.release();
284	int lastTokenIndex = tokenCountCurrent() - 1;	288	int lastTokenIndex = tokenCountCurrent() - 1;
285	int endOffset = (lastTokenIndex >= 0)	289	int endOffset = (lastTokenIndex >= 0)
286	? tokenOffset(lastTokenIndex) + existingToken(lastTokenIndex).length()	290	? tokenOffset(lastTokenIndex) + existingToken(lastTokenIndex).length()

Lines 423-428 Link Here

(-)lexer/src/org/netbeans/lib/lexer/inc/TokenListUpdater.java (+1 lines)
423	}	423	}
424	}	424	}
425	} while (relex);	425	} while (relex);
		426	lexerInputOperation.release();
426	}	427	}
427		428
428	// Now ensure that the original tokens will be replaced by the relexed ones.	429	// Now ensure that the original tokens will be replaced by the relexed ones.

Lines 136-139 Link Here

(-)lexer/src/org/netbeans/spi/lexer/Lexer.java (+10 lines)
136	*/	136	*/
137	Object state();	137	Object state();
138		138
		139	/**
		140	* Infrastructure calls this method when it no longer needs this lexer for lexing
		141	* so it becomes unused.
		142	* <br/>
		143	* If lexer instances are cached and reused later
		144	* then this method should first release all the references that might cause
		145	* memory leaks and then add this unused lexer to the cache.
		146	*/
		147	public void release();
		148
139	}	149	}

Lines 53-56 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/dump/TextAsSingleTokenLexer.java (+3 lines)
53	}	53	}
54	}	54	}
55		55
		56	public void release() {
		57	}
		58
56	}	59	}

Lines 187-192 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/dump/TokenDumpLexer.java (+3 lines)
187	return tokenFactory.createToken(id);	187	return tokenFactory.createToken(id);
188	}	188	}
189		189
		190	public void release() {
		191	}
		192
190	private static final class UnicodeCharValueProvider implements TokenPropertyProvider {	193	private static final class UnicodeCharValueProvider implements TokenPropertyProvider {
191		194
192	static final UnicodeCharValueProvider INSTANCE = new UnicodeCharValueProvider();	195	static final UnicodeCharValueProvider INSTANCE = new UnicodeCharValueProvider();

Lines 50-53 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleCharLexer.java (+3 lines)
50	return null; // always in default state after token recognition	50	return null; // always in default state after token recognition
51	}	51	}
52		52
		53	public void release() {
		54	}
		55
53	}	56	}

Lines 104-107 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleJavadocLexer.java (+3 lines)
104	return tokenFactory.createToken(id);	104	return tokenFactory.createToken(id);
105	}	105	}
106		106
		107	public void release() {
		108	}
		109
107	}	110	}

Lines 212-215 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleLexer.java (+3 lines)
212	: tokenFactory.createToken(id);	212	: tokenFactory.createToken(id);
213	}	213	}
214		214
		215	public void release() {
		216	}
		217
215	}	218	}

Lines 96-99 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimplePlainLexer.java (+3 lines)
96	return null; // always in default state after token recognition	96	return null; // always in default state after token recognition
97	}	97	}
98		98
		99	public void release() {
		100	}
		101
99	}	102	}

Lines 94-98 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleStringLexer.java (+3 lines)
94	private Token<SimpleStringTokenId> token(SimpleStringTokenId id) {	94	private Token<SimpleStringTokenId> token(SimpleStringTokenId id) {
95	return tokenFactory.createToken(id);	95	return tokenFactory.createToken(id);
96	}	96	}
		97
		98	public void release() {
		99	}
97		100
98	}	101	}

Lines 95-98 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/simple/SimpleStringTokenId.java (+3 lines)
95	return language;	95	return language;
96	}	96	}
97		97
		98	public void release() {
		99	}
		100
98	}	101	}




import junit.framework.TestCase;
import org.netbeans.api.lexer.InputAttributes;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.LanguagePath;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.api.lexer.TokenSequence;

        TokenHierarchy<?> hi = TokenHierarchy.get(doc);
        TokenSequence<? extends TokenId> ts = hi.tokenSequence();

        ts = hi.tokenSequence();
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts, StateTokenId.A, "a", 0);
        assertEquals(LexerTestUtilities.lookahead(ts), 0);

        } catch (IllegalStateException e) {
            // Expected fail of lexer
        }
    }

    public void testBatchLexerRelease() throws Exception {
        String text = "ab";
        InputAttributes attrs = new InputAttributes();
        TokenHierarchy<?> hi = TokenHierarchy.create(text, false, StateTokenId.language(),
                null, attrs);
        TokenSequence<? extends TokenId> ts = hi.tokenSequence();
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts, StateTokenId.A, "a", 0);
        assertTrue(ts.moveNext());
        LanguagePath lp = LanguagePath.get(StateTokenId.language());
        assertFalse(Boolean.TRUE.equals(attrs.getValue(lp, "lexerRelease")));
        LexerTestUtilities.assertTokenEquals(ts, StateTokenId.BMULTI, "b", 1);
        assertFalse(ts.moveNext());
        assertTrue(Boolean.TRUE.equals(attrs.getValue(lp, "lexerRelease")));

    }

    public void testIncLexerRelease() throws Exception {
        Document doc = new ModificationTextDocument();
        // Assign a language to the document
        InputAttributes attrs = new InputAttributes();
        doc.putProperty(InputAttributes.class, attrs);
        
        // Insert initial text into document
        String text = "ab";
        doc.insertString(0, text, null);
        // Put the language now into the document so that lexing starts from scratch
        doc.putProperty(Language.class, StateTokenId.language());
        TokenHierarchy<?> hi = TokenHierarchy.get(doc);
        TokenSequence<? extends TokenId> ts = hi.tokenSequence();

        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts, StateTokenId.A, "a", 0);
        LanguagePath lp = LanguagePath.get(StateTokenId.language());
        assertFalse(Boolean.TRUE.equals(attrs.getValue(lp, "lexerRelease")));
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts, StateTokenId.BMULTI, "b", 1);
        assertFalse(ts.moveNext());
        assertTrue(Boolean.TRUE.equals(attrs.getValue(lp, "lexerRelease")));
        attrs.setValue(lp, "lexerRelease", Boolean.FALSE, false);

        // Do modification and check lexer release after it
        doc.insertString(1, "b", null);
        assertTrue(Boolean.TRUE.equals(attrs.getValue(lp, "lexerRelease")));
    }

}

Lines 97-100 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/state/StateLexer.java (+7 lines)
97	return tokenFactory.createToken(id);	97	return tokenFactory.createToken(id);
98	}	98	}
99		99
		100	public void release() {
		101	InputAttributes attrs = info.inputAttributes();
		102	if (attrs != null) {
		103	attrs.setValue(StateTokenId.language(), "lexerRelease", Boolean.TRUE, false);
		104	}
		105	}
		106
100	}	107	}

Lines 886-889 Link Here

(-)web/el/lexer/src/org/netbeans/modules/el/lexer/ELLexer.java (+3 lines)
886	return tokenFactory.createToken(tokenId);	886	return tokenFactory.createToken(tokenId);
887	}	887	}
888		888
		889	public void release() {
		890	}
		891
889	}	892	}

Lines 986-991 Link Here

(-)web/jspsyntax/lexer/src/org/netbeans/lib/jsp/lexer/JspLexer.java (+2 lines)
986	return tokenFactory.createToken(tokenId);	986	return tokenFactory.createToken(tokenId);
987	}	987	}
988		988
		989	public void release() {
		990	}
989		991
990	}	992	}
991		993

Lines 897-900 Link Here

(-)xml/lexer/src/org/netbeans/lib/xml/lexer/XMLLexer.java (+4 lines)
897	private boolean isAF(int ch) {	897	private boolean isAF(int ch) {
898	return( (ch >= 'a' && ch <= 'f') \|\| (ch >= 'A' && ch <= 'F') );	898	return( (ch >= 'a' && ch <= 'f') \|\| (ch >= 'A' && ch <= 'F') );
899	}	899	}
		900
		901	public void release() {
		902	}
		903
900	}	904	}

Return to bug 89000

Lines 91-96 Link Here

(-)lexer/api/apichanges.xml (+14 lines)
91	<!-- ACTUAL CHANGES BEGIN HERE: -->	91	<!-- ACTUAL CHANGES BEGIN HERE: -->
92		92
93	<changes>	93	<changes>
		94	<change id="api.lexer.release">
		95	<api name="api"/>
		96	<summary>Added Lexer.release()</summary>
		97	<version major="1" minor="12"/>
		98	<date day="7" month="12" year="2006"/>
		99	<author login="mmetelka"/>
		100	<compatibility source="incompatible" deletion="no" addition="yes" modification="no"/>
		101	<description>
		102	<p>
		103	Added <code>Lexer.release()</code> useful for lexer instances caching.
		104	</p>
		105	</description>
		106	</change>
		107
94	<change id="api.customembedding">	108	<change id="api.customembedding">
95	<api name="api"/>	109	<api name="api"/>
96	<summary>Adding custom embedding creation TokenSequence.createEmbedding()</summary>	110	<summary>Adding custom embedding creation TokenSequence.createEmbedding()</summary>

Lines 17-22 Link Here

(-)lexer/test/unit/src/org/netbeans/lib/lexer/test/state/InvalidLexerOperationTest.java (-1 / +47 lines)
17	import junit.framework.TestCase;	17	import junit.framework.TestCase;
18	import org.netbeans.api.lexer.InputAttributes;	18	import org.netbeans.api.lexer.InputAttributes;
19	import org.netbeans.api.lexer.Language;	19	import org.netbeans.api.lexer.Language;
		20	import org.netbeans.api.lexer.LanguagePath;
20	import org.netbeans.api.lexer.TokenHierarchy;	21	import org.netbeans.api.lexer.TokenHierarchy;
21	import org.netbeans.api.lexer.TokenId;	22	import org.netbeans.api.lexer.TokenId;
22	import org.netbeans.api.lexer.TokenSequence;	23	import org.netbeans.api.lexer.TokenSequence;
Lines 54-60 Link Here
54	TokenHierarchy<?> hi = TokenHierarchy.get(doc);	55	TokenHierarchy<?> hi = TokenHierarchy.get(doc);
55	TokenSequence<? extends TokenId> ts = hi.tokenSequence();	56	TokenSequence<? extends TokenId> ts = hi.tokenSequence();
56		57
57	ts = hi.tokenSequence();
58	assertTrue(ts.moveNext());	58	assertTrue(ts.moveNext());
59	LexerTestUtilities.assertTokenEquals(ts, StateTokenId.A, "a", 0);	59	LexerTestUtilities.assertTokenEquals(ts, StateTokenId.A, "a", 0);
60	assertEquals(LexerTestUtilities.lookahead(ts), 0);	60	assertEquals(LexerTestUtilities.lookahead(ts), 0);
Lines 68-73 Link Here
68	} catch (IllegalStateException e) {	68	} catch (IllegalStateException e) {
69	// Expected fail of lexer	69	// Expected fail of lexer
70	}	70	}
		71	}
		72
		73	public void testBatchLexerRelease() throws Exception {
		74	String text = "ab";
		75	InputAttributes attrs = new InputAttributes();
		76	TokenHierarchy<?> hi = TokenHierarchy.create(text, false, StateTokenId.language(),
		77	null, attrs);
		78	TokenSequence<? extends TokenId> ts = hi.tokenSequence();
		79	assertTrue(ts.moveNext());
		80	LexerTestUtilities.assertTokenEquals(ts, StateTokenId.A, "a", 0);
		81	assertTrue(ts.moveNext());
		82	LanguagePath lp = LanguagePath.get(StateTokenId.language());
		83	assertFalse(Boolean.TRUE.equals(attrs.getValue(lp, "lexerRelease")));
		84	LexerTestUtilities.assertTokenEquals(ts, StateTokenId.BMULTI, "b", 1);
		85	assertFalse(ts.moveNext());
		86	assertTrue(Boolean.TRUE.equals(attrs.getValue(lp, "lexerRelease")));
		87
		88	}
		89
		90	public void testIncLexerRelease() throws Exception {
		91	Document doc = new ModificationTextDocument();
		92	// Assign a language to the document
		93	InputAttributes attrs = new InputAttributes();
		94	doc.putProperty(InputAttributes.class, attrs);
		95
		96	// Insert initial text into document
		97	String text = "ab";
		98	doc.insertString(0, text, null);
		99	// Put the language now into the document so that lexing starts from scratch
		100	doc.putProperty(Language.class, StateTokenId.language());
		101	TokenHierarchy<?> hi = TokenHierarchy.get(doc);
		102	TokenSequence<? extends TokenId> ts = hi.tokenSequence();
		103
		104	assertTrue(ts.moveNext());
		105	LexerTestUtilities.assertTokenEquals(ts, StateTokenId.A, "a", 0);
		106	LanguagePath lp = LanguagePath.get(StateTokenId.language());
		107	assertFalse(Boolean.TRUE.equals(attrs.getValue(lp, "lexerRelease")));
		108	assertTrue(ts.moveNext());
		109	LexerTestUtilities.assertTokenEquals(ts, StateTokenId.BMULTI, "b", 1);
		110	assertFalse(ts.moveNext());
		111	assertTrue(Boolean.TRUE.equals(attrs.getValue(lp, "lexerRelease")));
		112	attrs.setValue(lp, "lexerRelease", Boolean.FALSE, false);
		113
		114	// Do modification and check lexer release after it
		115	doc.insertString(1, "b", null);
		116	assertTrue(Boolean.TRUE.equals(attrs.getValue(lp, "lexerRelease")));
71	}	117	}
72		118
73	}	119	}