plugins: Add lookup cache plugin

One major weakness with Ansible's "lookup" plugins is that they are
evaluated _every single time they are used_, even indirectly.  This
means, for example, a shell command could be run many times, potentially
resulting in different values, or executing a complex calculation that
always provides the same result.  Ansible does not have a built-in way
to cache the result of a `lookup` or `query` call, so I created this
one.  It's inspired by [ansible-cached-lookup][0], which didn't actually
work and is apparently unmaintained.  Instead of using a hard-coded
file-based caching system, however, my plugin uses Ansible's
configuration and plugin infrastructure to store values with any
available cache plugin.

Although looking up the _pyrocufflink.net_ wildcard certificate with the
Kubernetes API isn't particularly expensive by itself right now, I can
envision several other uses that may be.  Having this plugin available
could speed up future playbooks.

[0]: https://pypi.org/project/ansible-cached-lookup

.gitignore

@@ -1,4 +1,5 @@
 /.inventory-cache
+/.lookup-cache
 /.vault-secret.gpg
 .fact-cache
 /secure.yaml

ansible.cfg

@@ -5,6 +5,7 @@ inventory = hosts, hosts.pyrocufflink.yml
 
 callback_plugins = plugins/callback
 inventory_plugins = plugins/inventory
+lookup_plugins = plugins/lookup
 
 gathering = smart
 fact_caching = jsonfile
@@ -20,3 +21,7 @@ server = https://ntfy.pyrocufflink.blue
 [ara]
 api_client = http
 api_server = https://ara.ansible.pyrocufflink.blue
+
+[lookup]
+cache_plugin = jsonfile
+cache_connection = .lookup-cache

group_vars/wildcard-cert.yml

@@ -1,21 +1,17 @@
-apache_ssl_certificate_data: >-
+pyrocufflink_wildcard_cert_secret: >-
-  {{
+  {{ lookup(
-  query(
+    "cache",
     "kubernetes.core.k8s",
     kind="Secret",
     namespace="default",
     resource_name="pyrocufflink-cert"
-  )[0].data["tls.crt"]
+  ) }}
-  | b64decode
-  }}
 
-apache_ssl_certificate_key_data: >-
+pyrocufflink_wildcard_cert: >-
-  {{
+  {{ pyrocufflink_wildcard_cert_secret.data["tls.crt"] | b64decode }}
-  query(
+
-    "kubernetes.core.k8s",
+pyrocufflink_wildcard_key: >-
-    kind="Secret",
+  {{ pyrocufflink_wildcard_cert_secret.data["tls.key"] | b64decode }}
-    namespace="default",
+
-    resource_name="pyrocufflink-cert"
+apache_ssl_certificate_data: "{{ pyrocufflink_wildcard_cert }}"
-  )[0].data["tls.key"]
+apache_ssl_certificate_key_data: "{{ pyrocufflink_wildcard_key }}"
-  | b64decode
-  }}

plugins/lookup/cache.py

@@ -0,0 +1,103 @@
+import functools
+import hashlib
+
+from ansible.constants import config
+from ansible.errors import AnsibleError
+from ansible.plugins.loader import cache_loader, lookup_loader
+from ansible.plugins.lookup import LookupBase, display
+
+DOCUMENTATION = """
+lookup: cache
+author: Dustin C. Hatch <dustin@hatch.name>
+options:
+  cache_plugin:
+    description:
+    - Cache plugin to use
+    type: str
+    required: false
+    default: memory
+    env:
+    - name: ANSIBLE_LOOKUP_CACHE_PLUGIN
+    ini:
+    - section: lookup
+      key: cache_plugin
+  cache_timeout:
+    description:
+    - Cache duration in seconds
+    default: 3600
+    type: int
+    env:
+    - name: ANSIBLE_LOOKUP_CACHE_TIMEOUT
+    ini:
+    - section: lookup
+      key: cache_timeout
+  cache_connection:
+    description:
+    - Cache connection data or path, read cache plugin documentation for specifics.
+    type: str
+    env:
+    - name: ANSIBLE_LOOKUP_CACHE_CONNECTION
+    ini:
+    - section: lookup
+      key: cache_connection
+  cache_prefix:
+    description:
+    - Prefix to use for cache plugin files/tables
+    default: ''
+    env:
+    - name: ANSIBLE_LOOKUP_CACHE_PREFIX
+    ini:
+    - section: lookup
+      key: cache_prefix
+"""
+
+
+def _get_option(key: str):
+    return config.get_config_value(
+        key, plugin_type='lookup', plugin_name='cache'
+    )
+
+
+@functools.cache
+def _get_cache():
+    cache_plugin = _get_option('cache_plugin')
+    cache_options = {}
+    if cache_connection := _get_option('cache_connection'):
+        cache_options['_uri'] = cache_connection
+    if cache_timeout := _get_option('cache_timeout'):
+        cache_options['_timeout'] = cache_timeout
+    if cache_prefix := _get_option('cache_prefix'):
+        cache_options['_prefix'] = cache_prefix
+    return cache_loader.get(cache_plugin, **cache_options)
+
+
+class LookupModule(LookupBase):
+    def run(self, terms, variables=None, **kwargs):
+        cache = _get_cache()
+        display.v(f'lookup cache: using cache plugin {cache.plugin_name}')
+        h = hashlib.sha1()
+        h.update(str((terms, kwargs)).encode('utf-8'))
+        key = h.hexdigest()
+        try:
+            result = cache.get(key)
+        except KeyError:
+            result = None
+        if result is None:
+            lookup_name, terms = terms[0], terms[1:]
+            lookup = lookup_loader.get(
+                lookup_name, loader=self._loader, templar=self._templar
+            )
+            if lookup is None:
+                raise AnsibleError(
+                    f'Could not find lookup plugin {lookup_name!r}'
+                )
+            result = lookup.run(terms, variables=variables, **kwargs)
+            cache.set(key, result)
+        else:
+            str_terms = ', '.join(repr(t) for t in terms)
+            str_kwargs = ', '.join(f'{k}={v!r}' for k, v in kwargs.items())
+            display.v(
+                'lookup cache: found cached value for'
+                f' lookup({str_terms}, {str_kwargs})'
+            )
+        return result