XML Lexer: Support ENTITY, NOTION, Enumerations with NMTokens for DTD editor

ide/xml.lexer/test/unit/data/goldenfiles/org/netbeans/api/xml/lexer/attrlist.pass

@@ -5,6 +5,7 @@ NAME, WS, OPERATOR, NAME, WS, OPERATOR, WS, NAME, OPERATOR, WS, STRING, WS, NAME
 NAME, WS, NAME, WS, KEYWORD, WS, NAME, WS, KEYWORD, WS, KEYWORD, WS
 NAME, WS, KEYWORD, WS, KEYWORD, WS, NAME, WS, KEYWORD, WS, KEYWORD, WS
 NAME, WS, KEYWORD, WS, KEYWORD, WS, NAME, WS, KEYWORD, WS, KEYWORD, WS
-NAME, WS, KEYWORD, WS, OPERATOR, NAME, OPERATOR, NAME, OPERATOR, WS, KEYWORD, WS, SYMBOL, PLAIN
+NAME, WS, KEYWORD, WS, OPERATOR, NAME, OPERATOR, NAME, OPERATOR, WS, KEYWORD, WS
+NAME, WS, OPERATOR, NAME, WS, OPERATOR, WS, NAME, WS, OPERATOR, WS, KEYWORD, WS, SYMBOL, PLAIN
 SYMBOL, DECLARATION, WS, REFERENCE, SYMBOL, PLAIN
 SYMBOL, DECLARATION, WS, NAME, WS, REFERENCE, SYMBOL, PLAIN

ide/xml.lexer/test/unit/data/goldenfiles/org/netbeans/api/xml/lexer/comment.pass

@@ -0,0 +1,3 @@
+COMMENT, PLAIN
+COMMENT
+PLAIN

ide/xml.lexer/test/unit/data/goldenfiles/org/netbeans/api/xml/lexer/embeddedEntities.pass

@@ -11,4 +11,4 @@ SYMBOL, DECLARATION, WS, NAME, WS, STRING, {ERROR}, STRING, SYMBOL, PLAIN
 SYMBOL, DECLARATION, WS, NAME, WS, STRING, {ERROR}, STRING, SYMBOL, PLAIN
 SYMBOL, DECLARATION, WS, NAME, WS, ERROR, SYMBOL, PLAIN
 SYMBOL, DECLARATION, WS, NAME, WS, ERROR, SYMBOL, PLAIN
-SYMBOL, DECLARATION, WS, NAME, WS, ERROR, ERROR, ERROR, SYMBOL, PLAIN
+SYMBOL, DECLARATION, WS, NAME, WS, ERROR, NAME, ERROR, SYMBOL, PLAIN

ide/xml.lexer/test/unit/data/goldenfiles/org/netbeans/api/xml/lexer/entity.pass

@@ -0,0 +1,7 @@
+SYMBOL, DECLARATION, WS, NAME, WS, STRING, WS, SYMBOL, PLAIN
+SYMBOL, DECLARATION, WS, NAME, WS, KEYWORD, WS, STRING, SYMBOL, PLAIN
+SYMBOL, DECLARATION, WS, NAME, WS, KEYWORD, WS, STRING, WS, STRING, SYMBOL, PLAIN
+SYMBOL, DECLARATION, WS, NAME, WS, KEYWORD, WS, STRING, WS, STRING, WS, KEYWORD, WS, NAME, SYMBOL, PLAIN
+SYMBOL, DECLARATION, WS, OPERATOR, WS, NAME, WS, STRING, WS, SYMBOL, PLAIN
+SYMBOL, DECLARATION, WS, OPERATOR, WS, NAME, WS, KEYWORD, WS, STRING, SYMBOL, PLAIN
+SYMBOL, DECLARATION, WS, OPERATOR, WS, NAME, WS, KEYWORD, WS, STRING, WS, STRING, SYMBOL, PLAIN

ide/xml.lexer/test/unit/data/goldenfiles/org/netbeans/api/xml/lexer/notion.pass

@@ -0,0 +1,3 @@
+SYMBOL, DECLARATION, WS, NAME, WS, KEYWORD, WS, STRING, SYMBOL, PLAIN
+SYMBOL, DECLARATION, WS, NAME, WS, KEYWORD, WS, STRING, WS, STRING, SYMBOL, PLAIN
+SYMBOL, DECLARATION, WS, NAME, WS, KEYWORD, WS, STRING, SYMBOL, PLAIN

ide/xml.lexer/test/unit/src/org/netbeans/api/xml/lexer/DTDTokenIdTest.java

@@ -52,8 +52,8 @@ private String lexDocument(String resourceName) throws Exception {
 
         ((AbstractDocument)document).readLock();
         try {
-            TokenHierarchy th = TokenHierarchy.get(document);
-            TokenSequence ts = th.tokenSequence();
+            TokenHierarchy<?> th = TokenHierarchy.get(document);
+            TokenSequence<DTDTokenId> ts = th.tokenSequence(DTDTokenId.language());
             //assert(ts.tokenCount() == expectedIds.length);
             int overLine = 0;
             while(ts.moveNext()) {
@@ -101,16 +101,21 @@ private void checkTokenSequence(String resource) throws Exception {
         String path = getClass().getName();
         path = path.substring(0, path.lastIndexOf('.')).replace('.', '/');
 
-        FileObject w = FileUtil.toFileObject(getWorkDir()).createData(resource + ".out");
-        File g = new File(getDataDir(), "goldenfiles/" + path + "/" + resource + ".pass");
-        try (OutputStream o = w.getOutputStream(); 
+        FileObject testOutput = FileUtil.toFileObject(getWorkDir()).createData(resource + ".out");
+        File goldenFile = new File(getDataDir(), "goldenfiles/" + path + "/" + resource + ".pass");
+        try (OutputStream o = testOutput.getOutputStream();
             OutputStreamWriter wr = new OutputStreamWriter(o)) {
             wr.write(text);
         }
-        File d = new File(getWorkDir(), resource + ".diff");
-        assertFile(FileUtil.toFile(w), g, d);
+        File diffFile = new File(getWorkDir(), resource + ".diff");
+        assertFile(FileUtil.toFile(testOutput), goldenFile, diffFile);
     }
 
+    @Override
+    protected Language getLanguage() {
+        return DTDTokenId.language();
+    }
+
     public void testXMLDeclaration() throws Exception {
         checkTokenSequence("textDeclarations");
     }
@@ -123,16 +128,19 @@ public void testAttrlist() throws Exception {
         checkTokenSequence("attrlist");
     }
 
-    @Override
-    protected Language getLanguage() {
-        return DTDTokenId.language();
-    }
-
     public void testEmbeddedEntityRefs() throws Exception {
         checkTokenSequence("embeddedEntities");
     }
-    
+
     public void testComments() throws Exception {
-
+        checkTokenSequence("comment");
+    }
+
+    public void testNotions() throws Exception {
+        checkTokenSequence("notion");
+    }
+
+    public void testEntities() throws Exception {
+        checkTokenSequence("entity");
     }
 }

ide/xml.lexer/test/unit/src/org/netbeans/api/xml/lexer/resources/attrlist.dtd

@@ -5,6 +5,7 @@
   entities  ENTITIES    #IMPLIED                                id        ID          #IMPLIED
   idref     IDREF       #IMPLIED                                idrefs    IDREFS      #IMPLIED
   nmtoken   NMTOKEN     #IMPLIED                                nmtokens  NMTOKENS    #IMPLIED
-  notation  NOTATION    (pdf|dvi) #IMPLIED >
+  notation  NOTATION    (pdf|dvi) #IMPLIED
+  year (2025 | 2026 ) #REQUIRED >
 <!ATTLIST %entity;>
 <!ATTLIST name %parsedEntity;>

ide/xml.lexer/test/unit/src/org/netbeans/api/xml/lexer/resources/comment.dtd

@@ -0,0 +1,4 @@
+<!-- Demo Comment -->
+<!-- Demo
+  Comment
+-->

ide/xml.lexer/test/unit/src/org/netbeans/api/xml/lexer/resources/entity.dtd

@@ -0,0 +1,7 @@
+<!ENTITY ProjectName 'Apache NetBeans' >
+<!ENTITY ProjectLogo SYSTEM "https://netbeans.apache.org/_/images/apache-netbeans.svg">
+<!ENTITY ProjectLogo PUBLIC "DEMO // PUBLIC" "https://netbeans.apache.org/_/images/apache-netbeans.svg">
+<!ENTITY ProjectLogo PUBLIC "DEMO // PUBLIC" "https://netbeans.apache.org/_/images/apache-netbeans.svg" NDATA gif>
+<!ENTITY % ProjectName 'Apache NetBeans' >
+<!ENTITY % ProjectLogo SYSTEM "https://netbeans.apache.org/_/images/apache-netbeans.svg">
+<!ENTITY % ProjectLogo PUBLIC "DEMO // PUBLIC" "https://netbeans.apache.org/_/images/apache-netbeans.svg">

ide/xml.lexer/test/unit/src/org/netbeans/api/xml/lexer/resources/notion.dtd

@@ -0,0 +1,3 @@
+<!NOTATION simple PUBLIC "Public Simple">
+<!NOTATION simple PUBLIC "Public Simple" "system/simple">
+<!NOTATION simple SYSTEM "system/simple">

ide/xml.text/build.xml

@@ -21,4 +21,16 @@
 -->
 <project name="ide/xml.text" default="build" basedir=".">
     <import file="../../nbbuild/templates/projectized.xml"/>
+
+    <target name="javacc-syntax" depends="-download.release.files">
+        <mkdir dir="build"/>
+        <copy file="external/javacc-7.0.10.jar" tofile="build/javacc.jar"/>
+        <javacc target="src/org/netbeans/modules/xml/text/syntax/javacc/DTDSyntax.jj"
+                javacchome="build">
+        </javacc>
+        <javacc target="src/org/netbeans/modules/xml/text/syntax/javacc/XMLSyntax.jj"
+                javacchome="build">
+        </javacc>
+        <delete file="src/org/netbeans/modules/xml/text/syntax/javacc/SimpleCharStream.java" />
+    </target>
 </project>

ide/xml.text/external/binaries-list

@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+7EFCA479324AD44EB42433EC61D365C46776474C net.java.dev.javacc:javacc:7.0.10

ide/xml.text/external/javacc-7.0.10-license.txt

@@ -0,0 +1,34 @@
+Name: JavaCC
+Version: 7.0.10
+Description: Parser generator for SQL parser.
+Origin: https://github.com/javacc/javacc/
+OSR: 3106
+License: BSD-javacc-7.0.10
+Type: compile-time
+
+Copyright (c) 2006, Sun Microsystems, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the Sun Microsystems, Inc. nor the names of its
+      contributors may be used to endorse or promote products derived from
+      this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.

ide/xml.text/src/org/netbeans/modules/xml/text/syntax/javacc/DTDSyntax.jj

@@ -54,14 +54,13 @@
 options {
 //      USER_CHAR_STREAM = true;
     UNICODE_INPUT = true;  // generate unicode aware code
-    OPTIMIZE_TOKEN_MANAGER = true;
     BUILD_PARSER = false;
     STATIC = false;
 }
 
 
 PARSER_BEGIN(DTDSyntax)
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information