class Asciidoctor::PreprocessorReader
Public: Methods for retrieving lines from AsciiDoc source files, evaluating preprocessor directives as each line is read off the Array of lines.
Attributes
Public Class Methods
Public: Initialize the PreprocessorReader object
# File lib/asciidoctor/reader.rb, line 607 def initialize document, data = nil, cursor = nil, opts = {} @document = document super data, cursor, opts include_depth_default = document.attributes.fetch('max-include-depth', 64).to_i include_depth_default = 0 if include_depth_default < 0 # track both absolute depth for comparing to size of include stack and relative depth for reporting @maxdepth = {:abs => include_depth_default, :rel => include_depth_default} @include_stack = [] @includes = document.catalog[:includes] @skipping = false @conditional_stack = [] @include_processor_extensions = nil end
Public Instance Methods
# File lib/asciidoctor/reader.rb, line 1167 def create_include_cursor file, path, lineno if ::String === file dir = ::File.dirname file elsif ::RUBY_ENGINE_OPAL dir = ::File.dirname(file = file.to_s) else dir = (dir = ::File.dirname file.path) == '' ? '/' : dir file = file.to_s end Cursor.new file, dir, path, lineno end
(see Asciidoctor::Reader#empty?)
# File lib/asciidoctor/reader.rb, line 710 def empty? peek_line ? false : true end
# File lib/asciidoctor/reader.rb, line 1194 def exceeded_max_depth? if (abs_maxdepth = @maxdepth[:abs]) > 0 && @include_stack.size >= abs_maxdepth @maxdepth[:rel] else false end end
(see Asciidoctor::Reader#has_more_lines?)
# File lib/asciidoctor/reader.rb, line 705 def has_more_lines? peek_line ? true : false end
# File lib/asciidoctor/reader.rb, line 1190 def include_depth @include_stack.size end
# File lib/asciidoctor/reader.rb, line 1307 def include_processors? if @include_processor_extensions.nil? if @document.extensions? && @document.extensions.include_processors? !!(@include_processor_extensions = @document.extensions.include_processors) else @include_processor_extensions = false end else @include_processor_extensions != false end end
Public: Override the Asciidoctor::Reader#peek_line method to pop the include stack if the last line has been reached and there's at least one include on the stack.
Returns the next line of the source data as a String if there are lines remaining in the current include context or a parent include context. Returns nothing if there are no more lines remaining and the include stack is empty.
# File lib/asciidoctor/reader.rb, line 722 def peek_line direct = false if (line = super) line elsif @include_stack.empty? nil else pop_include peek_line direct end end
# File lib/asciidoctor/reader.rb, line 1179 def pop_include if @include_stack.size > 0 @lines, @file, @dir, @path, @lineno, @maxdepth, @process_lines = @include_stack.pop # FIXME kind of a hack #Document::AttributeEntry.new('infile', @file).save_to_next_block @document #Document::AttributeEntry.new('indir', ::File.dirname(@file)).save_to_next_block @document @look_ahead = 0 nil end end
# File lib/asciidoctor/reader.rb, line 621 def prepare_lines data, opts = {} result = super # QUESTION should this work for AsciiDoc table cell content? Currently it does not. if @document && @document.attributes['skip-front-matter'] if (front_matter = skip_front_matter! result) @document.attributes['front-matter'] = front_matter.join LF end end if opts.fetch :condense, true result.shift && @lineno += 1 while (first = result[0]) && first.empty? result.pop while (last = result[-1]) && last.empty? end if opts[:indent] Parser.adjust_indentation! result, opts[:indent], (@document.attr 'tabsize') end result end
Internal: Preprocess the directive to conditionally include or exclude content.
Preprocess the conditional directive (ifdef, ifndef, ifeval, endif) under the cursor. If Reader is currently skipping content, then simply track the open and close delimiters of any nested conditional blocks. If Reader is not skipping, mark whether the condition is satisfied and continue preprocessing recursively until the next line of available content is found.
keyword - The conditional inclusion directive (ifdef, ifndef, ifeval, endif) target - The target, which is the name of one or more attributes that are
used in the condition (blank in the case of the ifeval directive)
delimiter - The conditional delimiter for multiple attributes ('+' means all
attributes must be defined or undefined, ',' means any of the attributes can be defined or undefined.
text - The text associated with this directive (occurring between the square brackets)
Used for a single-line conditional block in the case of the ifdef or ifndef directives, and for the conditional expression for the ifeval directive.
Returns a Boolean indicating whether the cursor should be advanced
# File lib/asciidoctor/reader.rb, line 753 def preprocess_conditional_directive keyword, target, delimiter, text # attributes are case insensitive target = target.downcase unless (no_target = target.empty?) # must have a target before brackets if ifdef or ifndef # must not have text between brackets if endif # skip line if it doesn't meet this criteria # QUESTION should we warn for these bogus declarations? return false if (no_target && (keyword == 'ifdef' || keyword == 'ifndef')) || (text && keyword == 'endif') if keyword == 'endif' if @conditional_stack.empty? logger.error message_with_context %(unmatched macro: endif::#{target}[]), :source_location => cursor elsif no_target || target == (pair = @conditional_stack[-1])[:target] @conditional_stack.pop @skipping = @conditional_stack.empty? ? false : @conditional_stack[-1][:skipping] else logger.error message_with_context %(mismatched macro: endif::#{target}[], expected endif::#{pair[:target]}[]), :source_location => cursor end return true end if @skipping skip = false else # QUESTION any way to wrap ifdef & ifndef logic up together? case keyword when 'ifdef' case delimiter when ',' # skip if no attribute is defined skip = target.split(',', -1).none? {|name| @document.attributes.key? name } when '+' # skip if any attribute is undefined skip = target.split('+', -1).any? {|name| !@document.attributes.key? name } else # if the attribute is undefined, then skip skip = !@document.attributes.key?(target) end when 'ifndef' case delimiter when ',' # skip if any attribute is defined skip = target.split(',', -1).any? {|name| @document.attributes.key? name } when '+' # skip if all attributes are defined skip = target.split('+', -1).all? {|name| @document.attributes.key? name } else # if the attribute is defined, then skip skip = @document.attributes.key?(target) end when 'ifeval' # the text in brackets must match an expression # don't honor match if it doesn't meet this criteria return false unless no_target && EvalExpressionRx =~ text.strip # NOTE save values eagerly for Ruby 1.8.7 compat lhs, op, rhs = $1, $2, $3 lhs = resolve_expr_val lhs rhs = resolve_expr_val rhs # regex enforces a restricted set of math-related operations if op == '!=' skip = lhs.send :==, rhs else skip = !(lhs.send op.to_sym, rhs) end end end # conditional inclusion block if keyword == 'ifeval' || !text @skipping = true if skip @conditional_stack << {:target => target, :skip => skip, :skipping => @skipping} # single line conditional inclusion else unless @skipping || skip replace_next_line text.rstrip # HACK push dummy line to stand in for the opening conditional directive that's subsequently dropped unshift '' # NOTE force line to be processed again if it looks like an include directive # QUESTION should we just call preprocess_include_directive here? @look_ahead -= 1 if text.start_with? 'include::' end end true end
Internal: Preprocess the directive to include lines from another document.
Preprocess the directive to include the target document. The scenarios are as follows:
If SafeMode is SECURE or greater, the directive is ignore and the include directive line is emitted verbatim.
Otherwise, if an include processor is specified pass the target and attributes to that processor and expect an Array of String lines in return.
Otherwise, if the max depth is greater than 0, and is not exceeded by the stack size, normalize the target path and read the lines onto the beginning of the Array of source data.
If none of the above apply, emit the include directive line verbatim.
target - The unsubstituted String name of the target document to include as specified in the
target slot of the include directive.
attrlist - An attribute list String, which is the text between the square brackets of the
include directive.
Returns a [Boolean] indicating whether the line under the cursor was changed. To skip over the directive, call shift and return true.
# File lib/asciidoctor/reader.rb, line 866 def preprocess_include_directive target, attrlist doc = @document if ((expanded_target = target).include? ATTR_REF_HEAD) && (expanded_target = doc.sub_attributes target, :attribute_missing => 'drop-line').empty? shift if (doc.attributes['attribute-missing'] || Compliance.attribute_missing) == 'skip' unshift %(Unresolved directive in #{@path} - include::#{target}[#{attrlist}]) end true elsif include_processors? && (ext = @include_processor_extensions.find {|candidate| candidate.instance.handles? expanded_target }) shift # FIXME parse attributes only if requested by extension ext.process_method[doc, self, expanded_target, (doc.parse_attributes attrlist, [], :sub_input => true)] true # if running in SafeMode::SECURE or greater, don't process this directive # however, be friendly and at least make it a link to the source document elsif doc.safe >= SafeMode::SECURE # FIXME we don't want to use a link macro if we are in a verbatim context replace_next_line %(link:#{expanded_target}[]) elsif (abs_maxdepth = @maxdepth[:abs]) > 0 if @include_stack.size >= abs_maxdepth logger.error message_with_context %(maximum include depth of #{@maxdepth[:rel]} exceeded), :source_location => cursor return end parsed_attrs = doc.parse_attributes attrlist, [], :sub_input => true inc_path, target_type, relpath = resolve_include_path expanded_target, attrlist, parsed_attrs return inc_path unless target_type inc_linenos = inc_tags = nil if attrlist if parsed_attrs.key? 'lines' inc_linenos = [] (split_delimited_value parsed_attrs['lines']).each do |linedef| if linedef.include? '..' from, to = linedef.split '..', 2 inc_linenos += (to.empty? || (to = to.to_i) < 0) ? [from.to_i, 1.0/0.0] : ::Range.new(from.to_i, to).to_a else inc_linenos << linedef.to_i end end inc_linenos = inc_linenos.empty? ? nil : inc_linenos.sort.uniq elsif parsed_attrs.key? 'tag' unless (tag = parsed_attrs['tag']).empty? || tag == '!' inc_tags = (tag.start_with? '!') ? { (tag.slice 1, tag.length) => false } : { tag => true } end elsif parsed_attrs.key? 'tags' inc_tags = {} (split_delimited_value parsed_attrs['tags']).each do |tagdef| if tagdef.start_with? '!' inc_tags[tagdef.slice 1, tagdef.length] = false else inc_tags[tagdef] = true end unless tagdef.empty? || tagdef == '!' end inc_tags = nil if inc_tags.empty? end end if inc_linenos inc_lines, inc_offset, inc_lineno = [], nil, 0 begin open(inc_path, 'rb') do |f| select_remaining = nil f.each_line do |l| inc_lineno += 1 if select_remaining || (::Float === (select = inc_linenos[0]) && (select_remaining = select.infinite?)) # NOTE record line where we started selecting inc_offset ||= inc_lineno inc_lines << l else if select == inc_lineno # NOTE record line where we started selecting inc_offset ||= inc_lineno inc_lines << l inc_linenos.shift end break if inc_linenos.empty? end end end rescue logger.error message_with_context %(include #{target_type} not readable: #{inc_path}), :source_location => cursor return replace_next_line %(Unresolved directive in #{@path} - include::#{expanded_target}[#{attrlist}]) end shift # FIXME not accounting for skipped lines in reader line numbering if inc_offset parsed_attrs['partial-option'] = true push_include inc_lines, inc_path, relpath, inc_offset, parsed_attrs end elsif inc_tags inc_lines, inc_offset, inc_lineno, tag_stack, tags_used, active_tag = [], nil, 0, [], ::Set.new, nil if inc_tags.key? '**' if inc_tags.key? '*' select = base_select = (inc_tags.delete '**') wildcard = inc_tags.delete '*' else select = base_select = wildcard = (inc_tags.delete '**') end else select = base_select = !(inc_tags.value? true) wildcard = inc_tags.delete '*' end begin open(inc_path, 'rb') do |f| dbl_co, dbl_sb = '::', '[]' encoding = ::Encoding::UTF_8 if COERCE_ENCODING f.each_line do |l| inc_lineno += 1 # must force encoding since we're performing String operations on line l.force_encoding encoding if encoding if (l.include? dbl_co) && (l.include? dbl_sb) && TagDirectiveRx =~ l if $1 # end tag if (this_tag = $2) == active_tag tag_stack.pop active_tag, select = tag_stack.empty? ? [nil, base_select] : tag_stack[-1] elsif inc_tags.key? this_tag include_cursor = create_include_cursor inc_path, expanded_target, inc_lineno if (idx = tag_stack.rindex {|key, _| key == this_tag }) idx == 0 ? tag_stack.shift : (tag_stack.delete_at idx) logger.warn message_with_context %(mismatched end tag (expected '#{active_tag}' but found '#{this_tag}') at line #{inc_lineno} of include #{target_type}: #{inc_path}), :source_location => cursor, :include_location => include_cursor else logger.warn message_with_context %(unexpected end tag '#{this_tag}' at line #{inc_lineno} of include #{target_type}: #{inc_path}), :source_location => cursor, :include_location => include_cursor end end elsif inc_tags.key?(this_tag = $2) tags_used << this_tag # QUESTION should we prevent tag from being selected when enclosing tag is excluded? tag_stack << [(active_tag = this_tag), (select = inc_tags[this_tag]), inc_lineno] elsif !wildcard.nil? select = active_tag && !select ? false : wildcard tag_stack << [(active_tag = this_tag), select, inc_lineno] end elsif select # NOTE record the line where we started selecting inc_offset ||= inc_lineno inc_lines << l end end end rescue logger.error message_with_context %(include #{target_type} not readable: #{inc_path}), :source_location => cursor return replace_next_line %(Unresolved directive in #{@path} - include::#{expanded_target}[#{attrlist}]) end unless tag_stack.empty? tag_stack.each do |tag_name, _, tag_lineno| logger.warn message_with_context %(detected unclosed tag '#{tag_name}' starting at line #{tag_lineno} of include #{target_type}: #{inc_path}), :source_location => cursor, :include_location => (create_include_cursor inc_path, expanded_target, tag_lineno) end end unless (missing_tags = inc_tags.keys.to_a - tags_used.to_a).empty? logger.warn message_with_context %(tag#{missing_tags.size > 1 ? 's' : ''} '#{missing_tags.join ', '}' not found in include #{target_type}: #{inc_path}), :source_location => cursor end shift if inc_offset parsed_attrs['partial-option'] = true unless base_select && wildcard && inc_tags.empty? # FIXME not accounting for skipped lines in reader line numbering push_include inc_lines, inc_path, relpath, inc_offset, parsed_attrs end else begin # NOTE read content first so that we only advance cursor if IO operation succeeds inc_content = target_type == :file ? ::File.open(inc_path, 'rb') {|f| f.read } : open(inc_path, 'rb') {|f| f.read } shift push_include inc_content, inc_path, relpath, 1, parsed_attrs rescue logger.error message_with_context %(include #{target_type} not readable: #{inc_path}), :source_location => cursor return replace_next_line %(Unresolved directive in #{@path} - include::#{expanded_target}[#{attrlist}]) end end true end end
# File lib/asciidoctor/reader.rb, line 643 def process_line line return line unless @process_lines if line.empty? @look_ahead += 1 return line end # NOTE highly optimized if line.end_with?(']') && !line.start_with?('[') && line.include?('::') if (line.include? 'if') && ConditionalDirectiveRx =~ line # if escaped, mark as processed and return line unescaped if $1 == '\\' @unescape_next_line = true @look_ahead += 1 line.slice 1, line.length elsif preprocess_conditional_directive $2, $3, $4, $5 # move the pointer past the conditional line shift # treat next line as uncharted territory nil else # the line was not a valid conditional line # mark it as visited and return it @look_ahead += 1 line end elsif @skipping shift nil elsif (line.start_with? 'inc', '\\inc') && IncludeDirectiveRx =~ line # if escaped, mark as processed and return line unescaped if $1 == '\\' @unescape_next_line = true @look_ahead += 1 line.slice 1, line.length # QUESTION should we strip whitespace from raw attributes in Substitutors#parse_attributes? (check perf) elsif preprocess_include_directive $2, $3 # peek again since the content has changed nil else # the line was not a valid include line and is unchanged # mark it as visited and return it @look_ahead += 1 line end else # NOTE optimization to inline super @look_ahead += 1 line end elsif @skipping shift nil else # NOTE optimization to inline super @look_ahead += 1 line end end
Public: Push source onto the front of the reader and switch the context based on the file, document-relative path and line information given.
This method is typically used in an IncludeProcessor to add source read from the target specified.
Examples
path = 'partial.adoc' file = File.expand_path path data = IO.read file reader.push_include data, file, path
Returns this Reader object.
# File lib/asciidoctor/reader.rb, line 1104 def push_include data, file = nil, path = nil, lineno = 1, attributes = {} @include_stack << [@lines, @file, @dir, @path, @lineno, @maxdepth, @process_lines] if (@file = file) # NOTE if file is not a string, assume it's a URI if ::String === file @dir = ::File.dirname file elsif ::RUBY_ENGINE_OPAL @dir = ::URI.parse ::File.dirname(file = file.to_s) else # NOTE this intentionally throws an error if URI has no path (@dir = file.dup).path = (dir = ::File.dirname file.path) == '/' ? '' : dir file = file.to_s end path ||= ::File.basename file # only process lines in AsciiDoc files @process_lines = ASCIIDOC_EXTENSIONS[::File.extname file] else @dir = '.' # we don't know what file type we have, so assume AsciiDoc @process_lines = true end if path @path = path @includes[Helpers.rootname path] = attributes['partial-option'] ? nil : true if @process_lines else @path = '<stdin>' end @lineno = lineno if attributes.key? 'depth' depth = attributes['depth'].to_i depth = 1 if depth <= 0 @maxdepth = {:abs => (@include_stack.size - 1) + depth, :rel => depth} end # effectively fill the buffer if (@lines = prepare_lines data, :normalize => true, :condense => false, :indent => attributes['indent']).empty? pop_include else # FIXME we eventually want to handle leveloffset without affecting the lines if attributes.key? 'leveloffset' @lines.unshift '' @lines.unshift %(:leveloffset: #{attributes['leveloffset']}) @lines << '' if (old_leveloffset = @document.attr 'leveloffset') @lines << %(:leveloffset: #{old_leveloffset}) else @lines << ':leveloffset!:' end # compensate for these extra lines @lineno -= 2 end # FIXME kind of a hack #Document::AttributeEntry.new('infile', @file).save_to_next_block @document #Document::AttributeEntry.new('indir', @dir).save_to_next_block @document @look_ahead = 0 end self end
Private: Resolve the value of one side of the expression
Examples
expr = '"value"' resolve_expr_val expr # => "value" expr = '"value' resolve_expr_val expr # => "\"value" expr = '"{undefined}"' resolve_expr_val expr # => "" expr = '{undefined}' resolve_expr_val expr # => nil expr = '2' resolve_expr_val expr # => 2 @document.attributes['name'] = 'value' expr = '"{name}"' resolve_expr_val expr # => "value"
Returns The value of the expression, coerced to the appropriate type
# File lib/asciidoctor/reader.rb, line 1275 def resolve_expr_val val if ((val.start_with? '"') && (val.end_with? '"')) || ((val.start_with? '\'') && (val.end_with? '\'')) quoted = true val = val.slice 1, (val.length - 1) else quoted = false end # QUESTION should we substitute first? # QUESTION should we also require string to be single quoted (like block attribute values?) val = @document.sub_attributes val, :attribute_missing => 'drop' if val.include? ATTR_REF_HEAD if quoted val elsif val.empty? nil elsif val == 'true' true elsif val == 'false' false elsif val.rstrip.empty? ' ' elsif val.include? '.' val.to_f else # fallback to coercing to integer, since we # require string values to be explicitly quoted val.to_i end end
Internal: Resolve the target of an include directive.
An internal method to resolve the target of an include directive. This method must return an Array containing the resolved (absolute) path of the target, the target type (:file or :uri), and the path of the target relative to the outermost document. Alternately, the method may return a boolean to halt processing of the include directive line and to indicate whether the cursor should be advanced beyond this line (true) or the line should be reprocessed (false).
This method is overridden in Asciidoctor.js to resolve the target of an include in the browser environment.
target - A String containing the unresolved include target.
(Attribute references in target value have already been resolved).
attrlist - An attribute list String (i.e., the text between the square brackets). attributes - A Hash of attributes parsed from attrlist.
Returns An Array containing the resolved (absolute) include path, the target type, and the path relative to the outermost document. May also return a boolean to halt processing of the include.
# File lib/asciidoctor/reader.rb, line 1058 def resolve_include_path target, attrlist, attributes doc = @document if (Helpers.uriish? target) || (::String === @dir ? nil : (target = %(#{@dir}/#{target}))) return replace_next_line %(link:#{target}[#{attrlist}]) unless doc.attr? 'allow-uri-read' if doc.attr? 'cache-uri' # caching requires the open-uri-cached gem to be installed # processing will be automatically aborted if these libraries can't be opened Helpers.require_library 'open-uri/cached', 'open-uri-cached' unless defined? ::OpenURI::Cache elsif !::RUBY_ENGINE_OPAL # autoload open-uri ::OpenURI end [(::URI.parse target), :uri, target] else # include file is resolved relative to dir of current include, or base_dir if within original docfile inc_path = doc.normalize_system_path target, @dir, nil, :target_name => 'include file' unless ::File.file? inc_path if attributes.key? 'optional-option' shift return true else logger.error message_with_context %(include file not found: #{inc_path}), :source_location => cursor return replace_next_line %(Unresolved directive in #{@path} - include::#{target}[#{attrlist}]) end end # NOTE relpath is the path relative to the root document (or base_dir, if set) # QUESTION should we move relative_path method to Document relpath = doc.path_resolver.relative_path inc_path, doc.base_dir [inc_path, :file, relpath] end end
TODO Document this override also, we now have the field in the super class, so perhaps just implement the logic there?
# File lib/asciidoctor/reader.rb, line 1205 def shift if @unescape_next_line @unescape_next_line = false (line = super).slice 1, line.length else super end end
Private: Ignore front-matter, commonly used in static site generators
# File lib/asciidoctor/reader.rb, line 1220 def skip_front_matter! data, increment_linenos = true front_matter = nil if data[0] == '---' original_data = data.drop 0 data.shift front_matter = [] @lineno += 1 if increment_linenos while !data.empty? && data[0] != '---' front_matter << data.shift @lineno += 1 if increment_linenos end if data.empty? data.unshift(*original_data) @lineno = 0 if increment_linenos front_matter = nil else data.shift @lineno += 1 if increment_linenos end end front_matter end
Private: Split delimited value on comma (if found), otherwise semi-colon
# File lib/asciidoctor/reader.rb, line 1215 def split_delimited_value val (val.include? ',') ? (val.split ',') : (val.split ';') end
# File lib/asciidoctor/reader.rb, line 1319 def to_s %(#<#{self.class}@#{object_id} {path: #{@path.inspect}, line #: #{@lineno}, include depth: #{@include_stack.size}, include stack: [#{@include_stack.map {|inc| inc.to_s }.join ', '}]}>) end