Модуль:External data

Статья

на 29 ноября 2024 ([ источник])

full_url

{{#invoke:External data|full_url|url=/index.php?do=authors&author=637|base=https://www.apn.ru/index.php}} даст https://www.apn.ru/index.php?do=authors&author=637

---

--[[
	Обёртки над функциями расширения External Data
--]]

-- Dependencies:
local lang = mw.language.getContentLanguage ()
local find, split, gsub, trim
	= mw.ustring.find or string.find, mw.text.split or string.split, mw.ustring.gsub or string.gsub, mw.text.trim
local clone = mw.clone
local new_url, validate = mw.uri.new, mw.uri.validate
local dump, concat = mw.dumpObject, table.concat

-- Функции расширения ExternalData:
local fetch_pf, show_pf, loop_pf = '#get_web_data', '#external_value', '#display_external_table'

-- Слуюебные функции:
local function full_url_lua (url, base)
  	local parsed_url = new_url (url)
  	local parsed_base = new_url (base)
  	parsed_url.protocol = parsed_url.protocol or parsed_base.protocol or 'https'
  	parsed_url.authority = parsed_url.authority or parsed_base.authority or '??'
	return tostring (parsed_url)
end	-- local function full_url_lua (url, base)

-- Подключение к источнику данных:
local function connect (frame, query, format, mappings, cache, stale, encoding)
	local mappings_lines = {}
	for to, from in pairs (mappings) do
		if to and from then
			mappings_lines [#mappings_lines + 1] = to .. '=' .. from
		end
	end	
	local params = {
		format				= format
	  , url					= query .. '\n'
	  , encoding			= encoding
	  , data				= concat (mappings_lines, ',')
	  , ['cache seconds']	= cache and tostring (cache)
	}
	if format == 'XML' or format == 'HTML with XPath' then
		params [#params + 1] = 'use xpath'
	end
	if stale then
		params [#params + 1] = 'use stale cache'
	end
	-- Required by call_parser_function:
	if #params == 0 then
		params [1] = ''
	end
	local connect = frame:callParserFunction (fetch_pf, params)
	if connect and connect ~= '' then
		-- this is an error:
		return false, connect
	else
		-- successful connection:
		return query
	end
end	-- local function connect (frame, query, format, mappings, cache, stale, encoding)
	
local error_token = '^<span class="error">'
		
-- Однократое извлечение данных успешного подключения:
local function fetch (frame, query, format, mappings, cache, stale, encoding)
	local real_path, error = connect (frame, query, format, mappings, cache, stale, encoding)
	if real_path then
		local data = {}
		for key, _ in pairs (mappings) do
			local value = frame:callParserFunction (show_pf, key)
			if value and value ~= '' and not find (value, error_token) then
				data [key] = tonumber (value) or value
			end
		end	-- for key, _ in pairs (mappings)
		return data, real_path, lang:formatDate ('d xg Y', nil, true)
	else
		return nil, nil, lang:formatDate ('d xg Y', nil, true)
			 , error
	end	-- if real_path
end	-- local function fetch (frame, query, format, mappings, cache, stale, encoding)

local function valid_url (url)
	local valid, parsed = pcall (new_url, url)
	return valid and type (parsed) == 'table' and validate (parsed) and (parsed.host or parsed.path or parsed.query)
end

-- Множественное извлечение данных успешного подключения:
local function fetch_all (frame, query, format, mappings, cache, stale, template, next, encoding)
	if next and trim (next) ~= '' then
		mappings.next = next
	end
	local real_path, error = connect (frame, query, format, mappings, cache, stale, encoding)
	if real_path then
		local params = {}
		for key, _ in pairs (mappings) do
			params [#params + 1] = key .. '=' .. key
		end
		local separator = '(-*-)'
		local pf_params = {''
		  , template	= template
		  , data		= concat (params, ',')
		  , delimiter	= separator
		}
		local glued = frame:callParserFunction (loop_pf, pf_params)
		if glued and glued ~= '' and not find (glued, error_token) then
			local next_url = nil
			if next and trim (next) ~= '' then
				next_url = frame:callParserFunction (show_pf, 'next')
				if not valid_url (next_url) or find (next_url, error_token) then
					next_url = nil
				end
			end
			-- Normal return:
			return split (glued, separator, true), real_path, lang:formatDate ('d xg Y', nil, true), nil, next_url
		else
			error = (error or '')
		end
	end	-- if real_path
	return nil, nil, lang:formatDate ('d xg Y', nil, true), error -- failed.
end	-- local function fetch_all (frame, query, format, mappings, cache, stale, template, next, encoding)

local function scrape_links (frame)
  	local args = clone (frame.args)
  	local mappings = {}
  	for key, val in pairs (args) do
  		if not (key == 'url' or key == 'next' or key == 'format' or key == 'cache'
  			 or key == 'stale' or key == 'template' or key == 'encoding')
  		and val and trim (val) ~= '' then
  			mappings [key] = val
  		end
  	end
  	local next, visited, lines = args.url, {}, {}
  	local first, time
  	local pages, limit = 0, 50
  	while next and not visited [next] and pages <= limit do
  		local current, path, new_lines, error = next
	  	new_lines, path, time, error, next = fetch_all (
	  		frame
	  	  , full_url_lua (next, args.url)
	  	  , args.format
	  	  , mappings
	  	  , args.cache
	  	  , args.stale
	  	  , args.template
	  	  , args.next
	  	  , args.encoding
		)
	  	first = path or first
	  	if not error then
	  		pages = pages + 1
	  		for _, line in ipairs (new_lines) do
	  			lines [#lines + 1] = line
	  		end
	  	else
	  		lines [#lines + 1] = error
		end
		visited [current] = true		
	end	-- while next and not visited [next] and pages <= limit
	return frame:preprocess (
		concat (lines)
	 .. '\n{{!}}}\n'
	 .. '<div align="right">на ' .. time 
	 .. " ''([" .. gsub ((first or ''), '|' , '{{!}}') .. " источник])''</div>\n"
	)
end	-- local function scrape_links (frame)
	
local function full_url (frame)
	return full_url_lua (frame.args.url, frame.args.base)
end

local function trim (frame)	
	return tostring (frame.args.str)
end

return {
	fetch			= fetch
  , fetch_all		= fetch_all
  , scrape_links	= scrape_links
  , full_url		= full_url
  , trim			= trim
}