Refactor parsing to get rid of _saved_line

Handle the line-after-help-text case specially, which allows _has_tokens
(renamed to _reuse_tokens) to be used as the unget mechanism for help
texts as well, leaving _saved_line unused. Move the _reuse_tokens check
into _next_line().

This makes _parse_block() as straightforward as _parse_properties(), and
simplifies _parse_properties() a tiny bit too by getting rid of the
'_tokens_i = -1' assignment.

3 files changed, 48 insertions, 30 deletions

diff --git a/kconfiglib.py b/kconfiglib.py
index 982bbfb..00d00d0 100644
--- a/kconfiglib.py
+++ b/kconfiglib.py
@@ -754,10 +754,9 @@ class Kconfig(object):
         "_include_path",
         "_filestack",
         "_line",
-        "_saved_line",
         "_tokens",
         "_tokens_i",
-        "_has_tokens",
+        "_reuse_tokens",
     )
 
     #
@@ -931,9 +930,10 @@ class Kconfig(object):
         self.kconfig_filenames = [filename]
         self.env_vars = set()
 
-        # These implement a single line of "unget" for the parser
-        self._saved_line = None
-        self._has_tokens = False
+        # Used to avoid retokenizing lines when we discover that they're not
+        # part of the construct currently being parsed. This is kinda like an
+        # unget operation.
+        self._reuse_tokens = False
 
         # Keeps track of the location in the parent Kconfig files. Kconfig
         # files usually source other Kconfig files. See _enter_file().
@@ -1777,19 +1777,19 @@ class Kconfig(object):
         # Fetches and tokenizes the next line from the current Kconfig file.
         # Returns False at EOF and True otherwise.
 
-        # _saved_line provides a single line of "unget", currently only used
-        # for help texts.
-        #
-        # This also works as expected if _saved_line is "", indicating EOF:
-        # "" is falsy, and readline() returns "" over and over at EOF.
-        if self._saved_line:
-            self._line = self._saved_line
-            self._saved_line = None
-        else:
-            self._line = self._file.readline()
-            if not self._line:
-                return False
-            self._linenr += 1
+        # We might already have tokens from parsing a line and discovering that
+        # it's part of a different construct
+        if self._reuse_tokens:
+            self._reuse_tokens = False
+            self._tokens_i = -1
+            return True
+
+        # Note: readline() returns '' over and over at EOF, which we rely on
+        # for help texts at the end of files (see _line_after_help())
+        self._line = self._file.readline()
+        if not self._line:
+            return False
+        self._linenr += 1
 
         # Handle line joining
         while self._line.endswith("\\\n"):
@@ -1801,6 +1801,26 @@ class Kconfig(object):
 
         return True
 
+    def _line_after_help(self, line):
+        # Tokenizes the line after a help text. This case is special in that
+        # the line has already been fetched (to discover that it isn't part of
+        # the help text).
+        #
+        # An earlier version used a _saved_line variable instead that was
+        # checked in _next_line(). This special-casing gets rid of it and makes
+        # _reuse_tokens alone sufficient to handle unget.
+
+        if line:
+            # Handle line joining
+            while line.endswith("\\\n"):
+                line = line[:-2] + self._file.readline()
+                self._linenr += 1
+
+            self._tokens = self._tokenize(line)
+            self._reuse_tokens = True
+
+        self._line = line
+
 
     #
     # Tokenization
@@ -2427,12 +2447,7 @@ class Kconfig(object):
         # Returns the final menu node in the block (or 'prev' if the block is
         # empty). This allows chaining.
 
-        # We might already have tokens from parsing a line to check if it's a
-        # property and discovering it isn't. self._has_tokens functions as a
-        # kind of "unget".
-        while self._has_tokens or self._next_line():
-            self._has_tokens = False
-
+        while self._next_line():
             t0 = self._next_token()
             if t0 is None:
                 continue
@@ -2781,8 +2796,7 @@ class Kconfig(object):
 
             else:
                 # Reuse the tokens for the non-property line later
-                self._has_tokens = True
-                self._tokens_i = -1
+                self._reuse_tokens = True
                 return
 
     def _set_type(self, node, new_type):
@@ -2844,7 +2858,7 @@ class Kconfig(object):
                        " has 'help' but empty help text")
 
             node.help = ""
-            self._saved_line = line  # "Unget" the line
+            self._line_after_help(line)
             return
 
         # The help text goes on till the first non-empty line with less indent
@@ -2871,7 +2885,7 @@ class Kconfig(object):
         self._linenr += len(help_lines)
 
         node.help = "\n".join(help_lines).rstrip() + "\n"
-        self._saved_line = line  # "Unget" the line
+        self._line_after_help(line)
 
     def _parse_expr(self, transform_m):
         # Parses an expression from the tokens in Kconfig._tokens using a
diff --git a/tests/Klocation b/tests/Klocation
index 2604f5b..f997055 100644
--- a/tests/Klocation
+++ b/tests/Klocation
@@ -33,6 +33,8 @@ config HELP_3
       foo
       bar
     bool
+config \
+MULTI_DEF
 
 config MULTI_DEF
 
diff --git a/testsuite.py b/testsuite.py
index 802234e..c027189 100644
--- a/testsuite.py
+++ b/testsuite.py
@@ -1014,6 +1014,7 @@ g
         verify_locations(c.syms["MULTI_DEF"].nodes,
           "tests/Klocation:7",
           "tests/Klocation:37",
+          "tests/Klocation:39",
           "tests/Klocation_sourced:3",
           "tests/sub/Klocation_rsourced:2",
           "tests/sub/Klocation_gsourced1:1",
@@ -1024,7 +1025,7 @@ g
           "tests/sub/Klocation_grsourced2:1",
           "tests/sub/Klocation_grsourced1:1",
           "tests/sub/Klocation_grsourced2:1",
-          "tests/Klocation:70")
+          "tests/Klocation:72")
 
         verify_locations(c.named_choices["CHOICE"].nodes,
                          "tests/Klocation_sourced:5")
@@ -1103,7 +1104,8 @@ tests/Krecursive2:1
         [node.item.name for node in c.node_iter()
          if isinstance(node.item, Symbol)],
         ["SINGLE_DEF", "MULTI_DEF", "HELP_1", "HELP_2", "HELP_3", "MULTI_DEF",
-         "MULTI_DEF", "MENU_HOOK", "COMMENT_HOOK"] + 10*["MULTI_DEF"])
+         "MULTI_DEF", "MULTI_DEF", "MENU_HOOK", "COMMENT_HOOK"] + \
+        10*["MULTI_DEF"])
 
     verify_equal(
         [node.item.name for node in c.node_iter(True)