implemented a global exception handler for over.app.Main that uses introspection and docstrings to create human-friendly stack traces

implemented most of the command line parser
expanded ANSI color palette
added over.text.char_in_str
over.text.Output removes colors if the output stream is not a tty
minor fixes and improvements

over/app.py

@@ -1,13 +1,45 @@
 #! /usr/bin/env python3
 # encoding: utf-8
 
+# --------------------------------------------------
+# Library imports
 from collections import OrderedDict
 import enum
 import sys
+import re
+import traceback
+
+# --------------------------------------------------
+# Local imports
+from . import cmd
 from . import docs
 from . import text
 from . import types
 
+# --------------------------------------------------
+# Exceptions
+
+class UnknownAbbreviation(Exception):
+	"""
+	This abbreviation doesn't map to any known option.
+	"""
+	
+	pass
+
+class UnknownOption(Exception):
+	"""
+	This option was passed on the command line but is not known.
+	"""
+	
+	pass
+
+class ReadingUnsetOption(Exception):
+	"""
+	This option has no default, config file or command line value set.
+	"""
+	 
+	pass
+
 # --------------------------------------------------
 
 class Option_sources(enum.Enum):
@@ -17,42 +49,58 @@ class Option_sources(enum.Enum):
 	cmdline = 3
 
 class Option:
-	def __init__(self, name, description, callback, default=Option_sources.none, count=None, abbr=None, level=2, in_cfg_file=True, in_help=True):
+	def __init__(self, name, description, callback, default=Option_sources.none, count=0, overwrite=True, abbr=None, in_cfg_file=True, show_in_help=True):
 		self.name = name
 		self.description = description
 		self.callback = callback
 		self.default = default
 		self.count = count
+		self.overwrite = overwrite
 		self.abbr = abbr
-		self.level = level
 		self.in_cfg_file = in_cfg_file
-		self.in_help = in_help
+		self.show_in_help = show_in_help
 		
-		if default == Option_sources.none:
+		self.reset(True)
+	
+	def reset(self, default=False):
+		if not default or self.default == Option_sources.none:
 			self.source = Option_sources.none
-			self._value = Option_sources.none
+			self._value = None if self.overwrite else []
 		else:
 			self.source = Option_sources.default
-			self._value = default
+			self._value = self.default
 	
-	def set_value(self, value, source):
+	def set_value(self, raw_value, source, skip_callback=False):
-		self._value = value
+		"""
+		Adds a value to this option's state. If the `source` is different from `self.source`, resets the state.
+		
+		@action setting option value
+		"""
+		
+		if source != self.source:
+			self.reset()
+		
 		self.source = source
+		
+		if skip_callback:
+			value = raw_value
+		else:
+			if self.count == 1:
+				value = self.callback(raw_value[0])
+			else:
+				value = self.callback(raw_value)
+		
+		if self.overwrite:
+			self._value = value
+		else:
+			self._value.append(value)
 	
 	@property
 	def value(self):
 		if self.source == Option_sources.none:
-			raise AttributeError("option has no value")
+			raise ReadingUnsetOption("option --%s has no value" %(self.name))
 		else:
 			return self._value
-	
-	def dump(self):
-		if self.source == Option_sources.none:
-			return None
-		else:
-			if self.callback == bool:
-				if self.value:
-					...
 
 # --------------------------------------------------
 
@@ -65,6 +113,32 @@ class ConfigRouter:
 
 # --------------------------------------------------
 
+def expand_group(token, options):
+	"""
+	@action expanding a commandline group
+	"""
+	
+	output = []
+	
+	options_by_abbr = {option.abbr: option for option in options.values() if option.abbr}
+	
+	group_positive = token[0] is "+"
+	
+	for abbr in token[1:]:
+		try:
+			option = options_by_abbr[abbr]
+		except KeyError:
+			raise UnknownAbbreviation(abbr)
+		
+		if option.callback == bool:
+			output.append("--%s%s" %("" if group_positive else "no-", option.name))
+		else:
+			output.append("--%s" %(option.name))
+	
+	return output
+
+# --------------------------------------------------
+
 class Main:
 	"""
 	Application backbone. Provides:
@@ -73,21 +147,26 @@ class Main:
 	    * a configuration file that overrides the defaults
 	    * a command line parser that overrides all of the above
 	  * Help system that generates help text based on the application's configurable options.
+	  * A prettyprinting exception handler.
 	"""
 	
-	def __init__(self, name, version=None, license=None, use_cfg_file=False, auto_add_help=True):
+	def __init__(self, name, version=None, license=None, use_cfg_file=False, auto_add_help=True, handle_exceptions=True):
 		self.name = name
 		self.version = version
 		self.license = license
 		self.print = text.Output(name)
-		self.options = OrderedDict() # level: ndict
+		self.options = OrderedDict()
 		self.cfg = ConfigRouter(self.options)
+		self.targets = []
+		self.auto_add_help = auto_add_help
+		self.invocation = cmd.format_invocation(sys.argv)
 		
 		if use_cfg_file:
+			# ensure it exists, update it with new values, etc.
 			...
 		
-		if auto_add_help:
+		if handle_exceptions:
-			self.enable_help("help", "h")
+			sys.excepthook = self.exception_handler
 	
 	def __repr__(self):
 		return 'over.app.Main(name="%s")' %(self.name)
@@ -100,23 +179,26 @@ class Main:
 		sys.exit(rv)
 	
 	def add_option(self, *args, **kwargs):
+		"""
+		@action creating a new configuration option
+		"""
+		
 		option = Option(*args, **kwargs)
 		
-		if option.level not in self.options:
+		self.options[option.name] = option
-			self.options[option.level] = types.ndict()
-		
-		self.options[option.level][option.name] = option
 	
 	def add_doc(self, chapter, paragraphs):
 		...
 	
-	def enable_help(self, name, abbr):
+	def enable_help(self, name="help", abbr="h"):
 		"""
 		Map application help to --name and -abbr, and enable library ---help.
+		
+		@action adding help commandline options
 		"""
 		
 		self.add_option("help", "Display the application configuration view.", self.help, abbr=abbr, in_cfg_file=False)
-		self.add_option("help", "Display the library help and about views.", lambda: self.help(docs=docs.over_desc), level=3, in_cfg_file=False)
+		self.add_option("-help", "Display the library help and about views.", lambda: self.help(docs=docs.over_desc), in_cfg_file=False)
 	
 	def help(self, option=None, docs=None):
 		"""
@@ -131,18 +213,154 @@ class Main:
 		If `option` is set to an option's name, only that option will be described.
 		
 		If `docs` is set to something else, only those docs will be displayed.
+		
+		@action displaying help
 		"""
 		
-		...
+		self.print("called for help!")
 	
 	def parse_config(self):
 		...
 	
 	def parse_cmdline(self, cmdline=None):
+		"""
+		Parses arguments on the commandline.
+		
+		There are 5 types of arguments:
+		- bool flags (True is --option (+o), False is --no-option (-o))
+		- action options (passing the option triggers a callback, e.g. --execute)
+		- options taking a `Option.count` (>=1) arguments (--option ARGUMENT, --option ARG1 ARG2 ARG3 and so on)
+		- a guard that marks all following arguments as targets (--)
+		- all other arguments are targets
+		
+		Additionally, options are either over-writing or cumulative. An overwriting
+		option, if encountered more than once, takes the last (rightmost) state. E.g.
+		+A -A +A evaluates to True. A cumulative option would evaluate to a list of
+		[True, False, True].
+		
+		@action parsing commandline arguments
+		"""
+		
+		# todo update invocation
 		cmdline = cmdline or sys.argv[1:]
 		
-		...
+		# placing it here ensures --help is the last option in the list of options
+		if self.auto_add_help:
+			self.enable_help()
+		
+		last_read_option = None
+		remaining_payload_tokens = 0
+		temporary_payload = []
+		guard_passed = False
+		
+		for top_token in cmdline:
+			if not (remaining_payload_tokens or guard_passed) and len(top_token) >= 2 and top_token[0] in "+-" and top_token[1] != "-": # it's a group
+				expanded = expand_group(top_token, self.options)
+			else:
+				expanded = [top_token]
+			
+			for token in expanded:
+				if not (remaining_payload_tokens or guard_passed) and token[:2] == "--":
+					if token == "--":
+						guard_passed = True
+					else:
+						option_name = token[2:]
+						option_is_negative = False
+						
+						if option_name not in self.options:
+							if option_name[:3] == "no-":
+								option_name = option_name[3:]
+								option_is_negative = True
+						
+						try:
+							option = self.options[option_name]
+						except KeyError:
+							raise UnknownOption(token)
+						
+						last_read_option = option
+						
+						# booleans are handled specially
+						if option.callback == bool:
+							option.set_value(not option_is_negative, Option_sources.cmdline, skip_callback=True)
+						else:
+							remaining_payload_tokens = option.count
+							temporary_payload = []
+				else:
+					if remaining_payload_tokens:
+						temporary_payload.append(token)
+						remaining_payload_tokens -= 1
+						
+						if remaining_payload_tokens == 0:
+							last_read_option.set_value(temporary_payload, Option_sources.cmdline)
+					else:
+						self.targets.append(token)
 	
 	def parse(self, cmdline=None):
+		"""
+		@action assembling application configuration
+		"""
+		
 		self.parse_config()
 		self.parse_cmdline(cmdline)
+	
+	def exception_handler(self, exception_type, exception, trace):
+		"""
+		@action formatting a traceback
+		
+		Over Exception handler - prints human readable tracebacks.
+		"""
+		
+		self.print("<R>unhandled<.> <Y>%s<.> encountered" %(exception_type.__name__), self.print.tl.fail, end=":\n")
+		
+		# todo top level program name
+		# todo final level exception name and text
+		# fixme remove traceback
+		
+		tb_lines = ["", "---------------- Stack trace ----------------", "In program <W>%s<.>" %(self.invocation)]
+		trace = trace.tb_next
+		
+		while trace:
+			frame = trace.tb_frame
+			trace = trace.tb_next
+			method_name = frame.f_code.co_name
+			
+			# if there's a "self" in locals, we can get a hold of the method using getattr
+			if "self" in frame.f_locals:
+				method = getattr(frame.f_locals["self"], method_name)
+				method_display_name = "<c>%s<.>.<y>%s<.>" %(method.__self__.__class__.__name__, method_name)
+				method_type = "method"
+			
+			# if it's a global function, it's likely going to be here
+			elif method_name in frame.f_globals:
+				method = frame.f_globals[method_name]
+				method_display_name = "<y>%s<.>" %(method_name)
+				method_type = "function"
+			
+			# otherwise give up
+			else:
+				method = None
+				method_display_name = "<y>%s<.>" %(method_name)
+				method_type = "<r>unknown callable<.>"
+			
+			# use a docstring-provided action description if available
+			if method and method.__doc__ and "@action" in method.__doc__:
+				action = "while <m>%s<.> " %(re.findall("@action (.+)", method.__doc__)[0].strip())
+			else:
+				action = ""
+			
+			tb_lines.append("%sin %s <y>%s<.> at %s:%d," %(action, method_type, method_display_name, frame.f_code.co_filename, frame.f_lineno))
+		
+		reason = ": <R>%s<.>" %(" ".join(exception.args)) if exception.args else ""
+		tb_lines.append("exception <Y>%s<.> was raised%s (%s)" %(exception_type.__name__, reason, exception.__doc__.strip()))
+		
+		last_i = len(tb_lines) - 1
+		
+		for i, line in enumerate(tb_lines):
+			if i < 3:
+				format = "<t>"
+			elif i == last_i:
+				format = "%s ⇒ <i><t>" %((i - 3) * " ")
+			else:
+				format = "%s - <i><t>" %((i - 3) * " ")
+			
+			self.print(line, format=format, end="\n")

over/cmd.py

@@ -26,6 +26,9 @@ def char_in_str(chars, string):
 	
 	return False
 
+def format_invocation(args):
+	return " ".join(('"%s"' %(a) if char_in_str(" $()[];\\", a) else a) for a in args)
+
 class Command:
 	"""
 	A shell command with argument substitution and output capture.
@@ -83,7 +86,7 @@ class Command:
 				out.append(str(item))
 		
 		if pretty:
-			return [('"%s"' %(a) if char_in_str(" ()[];\\", a) else a) for a in out]
+			return [('"%s"' %(a) if char_in_str(" $()[];\\", a) else a) for a in out]
 		else:
 			return out
 	

over/file.py

@@ -10,6 +10,8 @@ def touch(path, times=None):
 	Sets a `path`'s atime and mtime.
 	
 	`times` is a tuple of (atime, mtime) and defaults to "now".
+	
+	@action touching a file
 	"""
 	
 	path = os.path.expanduser(path)
@@ -22,6 +24,8 @@ def touch(path, times=None):
 def count_lines(path):
 	"""
 	A reasonably fast and memory-lean line counter.
+	
+	@action counting lines in a file
 	"""
 	
 	lines = 0

over/misc.py

@@ -10,6 +10,8 @@ def import_module(path):
 	Imports a python file as a module. The `path` can be relative or absolute.
 	
 	Based on the work of Yuval Greenfield released into the public domain.
+	
+	@action importing a module
 	"""
 	
 	import imp
@@ -98,6 +100,8 @@ def hexdump(data, indent=0, offset=16, show_header=True, show_offsets=True, show
 	If `show_ascii` is True, each line is suffixed with its ASCII representation. Unprintable characters
 	are replaced with a dot.
 	The `output` must implement a .write(str x) method. If `output` is None, the string is returned instead.
+	
+	@action creating a hex dump
 	"""
 	
 	import io

over/template.py

@@ -1,11 +1,13 @@
 #! /usr/bin/env python3
 # encoding: utf-8
 
-# library imports
+# --------------------------------------------------
+# Library imports
 import over
 prefix = over.text.prefix
 
-# local imports
+# --------------------------------------------------
+# Local imports
 import version
 
 # --------------------------------------------------

over/text.py

@@ -1,6 +1,8 @@
 #! /usr/bin/env python3
 # encoding: utf-8
 
+# --------------------------------------------------
+# Library imports
 import datetime
 import fcntl
 import math
@@ -114,15 +116,24 @@ class Unit:
 
 # --------------------------------------------------
 
-colortags = {
+ansi_colors = {
-	"r": "\x1b[31;01m",
+	"B": ("\x1b[1;34m", "bright blue"),
-	"g": "\x1b[32;01m",
+	"C": ("\x1b[1;36m", "bright cyan"),
-	"y": "\x1b[33;01m",
+	"G": ("\x1b[1;32m", "bright green"),
-	"b": "\x1b[34;01m",
+	"K": ("\x1b[1;30m", "bright black"),
-	"m": "\x1b[35;01m",
+	"M": ("\x1b[1;35m", "bright magenta"),
-	"c": "\x1b[36;01m",
+	"R": ("\x1b[1;31m", "bright red"),
-	"B": "\x1b[01m",
+	"W": ("\x1b[1;37m", "bright white"),
-	".": "\x1b[39;49;00m" # reset
+	"Y": ("\x1b[1;33m", "bright yellow"),
+	"b": ("\x1b[0;34m", "blue"),
+	"c": ("\x1b[0;36m", "cyan"),
+	"g": ("\x1b[0;32m", "green"),
+	"k": ("\x1b[0;30m", "black"),
+	"m": ("\x1b[0;35m", "magenta"),
+	"r": ("\x1b[0;31m", "red"),
+	"w": ("\x1b[0;37m", "white"),
+	"y": ("\x1b[0;33m", "yellow"),
+	".": ("\x1b[0m", "reset"),
 }
 
 def render(text, colors=True):
@@ -131,8 +142,10 @@ def render(text, colors=True):
 	removes them (with colors=False) or replaces
 	them with terminal color codes.
 	
-	Color tags are <x> where x is the color code from colortags.
+	Color tags are <x> where x is the color code from ansi_colors.
 	<.> resets the color. Use <<x> for a literal <x>.
+	
+	@action coloring text
 	"""
 	
 	text = str(text)
@@ -151,10 +164,10 @@ def render(text, colors=True):
 		if window[0] == "<" and window[2] == ">":
 			code = window[1]
 			
-			if code in colortags.keys():
+			if code in ansi_colors.keys():
 				if not output or output[-1] != "<":
 					if colors:
-						output.append(colortags[code])
+						output.append(ansi_colors[code][0])
 					
 					window.clear()
 				elif output and output[-1] == "<":
@@ -189,6 +202,8 @@ def paragraph(text, width=0, indent=0, stamp=None):
 	
 	Formats text into an indented paragraph that fits the terminal and returns it.
 	Correctly handles colors.
+	
+	@action formatting a paragraph
 	"""
 	
 	fit_into_width = (width or get_terminal_size()[1]) - indent
@@ -236,6 +251,19 @@ def strlen(string):
 
 # --------------------------------------------------
 
+def char_in_str(chars, string):
+	"""
+	Returns True iff any char from `chars` is in `string`.
+	"""
+	
+	for char in chars:
+		if char in string:
+			return True
+	
+	return False
+
+# --------------------------------------------------
+
 def count_leading(text, char):
 	"""
 	Returns the count of leading `char`s within `text`.
@@ -256,20 +284,20 @@ class Output:
 	class tag:
 		class short:
 			info = "   "
-			debug = "  <b>?<.>"
+			debug = "  <c>?<.>"
-			start = "<B>>>><.>"
+			start = "<W>>>><.>"
 			exec = start
-			warn = "  <y>#<.>"
+			warn = "  <Y>#<.>"
-			fail = "<r>!!!<.>"
+			fail = "<R>!!!<.>"
-			done = "  <g>*<.>"
+			done = "  <G>*<.>"
 		class long:
 			info = "INFO"
-			debug = "<b>DEBG<.>"
+			debug = "<c>DEBG<.>"
-			start = "<B>EXEC<.>"
+			start = "<W>EXEC<.>"
 			exec = start
-			warn = "<y>WARN<.>"
+			warn = "<Y>WARN<.>"
-			fail = "<r>FAIL<.>"
+			fail = "<R>FAIL<.>"
-			done = "<g>DONE<.>"
+			done = "<G>DONE<.>"
 	
 	ts = tag.short
 	tl = tag.long
@@ -280,6 +308,8 @@ class Output:
 	Prints messages to the stdout with optional eye candy
 	like colors and timestamps.
 	
+	If the output stream is not a tty, colors are disabled.
+	
 	Formatting
 	==========
 	You can use all formatting characters supported by strftime, as well as:
@@ -292,12 +322,16 @@ class Output:
 	def __init__(self, name, format="[%Y-%m-%d %H:%M:%S] <T> -- <n>, <i><t>", colors=True, end=".\n", indent=True, stream=sys.stderr):
 		self.name = name
 		self.format = format
-		self.colors = colors
+		self.colors = colors and stream.isatty()
 		self.end = end
 		self.indent = indent
 		self.stream = stream
 	
-	def __call__(self, text, tag=None, format=None, colors=None, end=None):
+	def print(self, text, tag=None, format=None, colors=None, end=None):
+		"""
+		@action displaying text
+		"""
+		
 		tag = tag or self.__class__.tag.long.info
 		format = format or self.format
 		colors = colors or self.colors
@@ -307,7 +341,7 @@ class Output:
 		output = output.replace("<T>", tag)
 		output = output.replace("<n>", self.name)
 		
-		if self.indent:
+		if self.indent and "<i>" in format:
 			indent_width = render(output, colors=False).index("<i>")
 			text = paragraph(text, indent=indent_width)[indent_width:]
 		
@@ -317,6 +351,9 @@ class Output:
 		
 		self.stream.write(render(output, colors))
 		self.stream.flush()
+	
+	def __call__(self, *args, **kwargs):
+		self.print(*args, **kwargs)
 
 # --------------------------------------------------
 
@@ -460,6 +497,8 @@ class ProgressBar:
 		allowed to exceed this value. It can either be a number or a callable. If
 		it"s a callable, it will be called without arguments and shall return a number.
 		"precision" is the displayed floating point precision. Use 0 to force an integer.
+		
+		@action creating a ProgressBar
 		"""
 		
 		self.format = format
@@ -483,6 +522,8 @@ class ProgressBar:
 	def render(self, clear=True, stream=sys.stderr):
 		"""
 		Renders the progressbar into a stream. If clear is True, clears the previous content first.
+		
+		@action drawing a progressbar update
 		"""
 		
 		width = self.width or get_terminal_size()[1]
@@ -599,6 +640,21 @@ class ProgressBar:
 # --------------------------------------------------
 
 if __name__ == "__main__":
+	o = Output("over.text")
+	o("Sample info message")
+	o("Sample debug message", o.tl.debug)
+	o("Sample action start message", o.tl.start)
+	o("Sample action success message", o.tl.done)
+	o("Sample warning message", o.tl.warn)
+	o("Sample error message", o.tl.fail)
+	
+	o("Available colors", end=":\n")
+	
+	for abbr, (code, name) in sorted(ansi_colors.items()):
+		o("%s = <%s>%s<.>" %(abbr, abbr, name))
+	
+	o("ProgressBar test")
+	
 	pb = ProgressBar(
 		"§%a (§ra/§za) [§=a>§ A] §sa [§rb/§zb] (ETA §TA)",
 		{

over/types.py

@@ -21,6 +21,10 @@ class ndict(OrderedDict):
 	"""
 	
 	def __getattr__(self, name):
+		"""
+		@action looking up an attribute
+		"""
+		
 		if name in self:
 			return self[name]
 		elif name.replace("_", "-") in self:

over/types.pyx

@@ -21,6 +21,10 @@ cdef class ndict(OrderedDict):
 	"""
 	
 	def __getattr__(self, str name):
+		"""
+		@action looking up an attribute
+		"""
+		
 		if name in self:
 			return self[name]
 		elif name.replace("_", "-") in self:

over/version.py

@@ -4,5 +4,5 @@
 major = 0 # VERSION_MAJOR_IDENTIFIER
 minor = 0 # VERSION_MINOR_IDENTIFIER
 # VERSION_LAST_MM 0.0
-patch = 2 # VERSION_PATCH_IDENTIFIER
+patch = 3 # VERSION_PATCH_IDENTIFIER
 str = ".".join(str(v) for v in (major, minor, patch))