From 31e53046bd35b83027f8a8e1ab99a6eceb4e6a3c Mon Sep 17 00:00:00 2001 From: icebaker Date: Mon, 8 Jan 2024 21:41:30 -0300 Subject: adding support to markdown cartridges --- Gemfile.lock | 2 ++ components/storage.rb | 24 ++++++++++------ controllers/cartridges.rb | 28 +++++++++++-------- controllers/instance.rb | 7 ++--- logic/cartridge/parser.rb | 56 +++++++++++++++++++++++++++++++++++++ nano-bots.gemspec | 1 + ports/dsl/nano-bots.rb | 3 +- ports/dsl/nano-bots/cartridges.rb | 15 ++++++++++ spec/data/cartridges/markdown.md | 37 ++++++++++++++++++++++++ spec/logic/cartridge/parser_spec.rb | 31 ++++++++++++++++++++ 10 files changed, 178 insertions(+), 26 deletions(-) create mode 100644 logic/cartridge/parser.rb create mode 100644 ports/dsl/nano-bots/cartridges.rb create mode 100644 spec/data/cartridges/markdown.md create mode 100644 spec/logic/cartridge/parser_spec.rb diff --git a/Gemfile.lock b/Gemfile.lock index 6b3cc60..fc0da2e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,6 +13,7 @@ PATH pry (~> 0.14.2) rainbow (~> 3.1, >= 3.1.1) rbnacl (~> 7.1, >= 7.1.1) + redcarpet (~> 3.6) ruby-openai (~> 6.3, >= 6.3.1) sweet-moon (~> 0.0.7) @@ -82,6 +83,7 @@ GEM rainbow (3.1.1) rbnacl (7.1.1) ffi + redcarpet (3.6.0) regexp_parser (2.8.3) rexml (3.2.6) rspec (3.12.0) diff --git a/components/storage.rb b/components/storage.rb index 6a3fe13..839cd02 100644 --- a/components/storage.rb +++ b/components/storage.rb @@ -8,6 +8,8 @@ require_relative 'crypto' module NanoBot module Components class Storage + EXTENSIONS = %w[yml yaml markdown mdown mkdn md].freeze + def self.end_user(cartridge, environment) user = ENV.fetch('NANO_BOTS_END_USER', nil) @@ -74,11 +76,11 @@ module NanoBot def self.cartridge_path(path) partial = File.join(File.dirname(path), File.basename(path, File.extname(path))) - candidates = [ - path, - "#{partial}.yml", - "#{partial}.yaml" - ] + candidates = [path] + + EXTENSIONS.each do |extension| + candidates << "#{partial}.#{extension}" + end unless ENV.fetch('NANO_BOTS_CARTRIDGES_DIRECTORY', nil).nil? directory = ENV.fetch('NANO_BOTS_CARTRIDGES_DIRECTORY').sub(%r{/$}, '') @@ -88,8 +90,10 @@ module NanoBot partial = partial.sub(%r{^\.?/}, '') candidates << "#{directory}/#{partial}" - candidates << "#{directory}/#{partial}.yml" - candidates << "#{directory}/#{partial}.yaml" + + EXTENSIONS.each do |extension| + candidates << "#{directory}/#{partial}.#{extension}" + end end directory = "#{user_home!.sub(%r{/$}, '')}/.local/share/nano-bots/cartridges" @@ -99,8 +103,10 @@ module NanoBot partial = partial.sub(%r{^\.?/}, '') candidates << "#{directory}/#{partial}" - candidates << "#{directory}/#{partial}.yml" - candidates << "#{directory}/#{partial}.yaml" + + EXTENSIONS.each do |extension| + candidates << "#{directory}/#{partial}.#{extension}" + end candidates = candidates.uniq diff --git a/controllers/cartridges.rb b/controllers/cartridges.rb index df474a9..7215a99 100644 --- a/controllers/cartridges.rb +++ b/controllers/cartridges.rb @@ -3,16 +3,21 @@ require_relative '../components/storage' require_relative '../logic/helpers/hash' require_relative '../logic/cartridge/default' +require_relative '../logic/cartridge/parser' module NanoBot module Controllers class Cartridges + def self.load(path) + Logic::Cartridge::Parser.parse(File.read(path), format: File.extname(path)) + end + def self.all files = {} path = Components::Storage.cartridges_path - Dir.glob("#{path}/**/*.{yml,yaml}").each do |file| + Dir.glob("#{path}/**/*.{yml,yaml,markdown,mdown,mkdn,md}").each do |file| files[Pathname.new(file).realpath] = { base: path, path: Pathname.new(file).realpath @@ -22,16 +27,17 @@ module NanoBot cartridges = [] files.values.uniq.map do |file| - cartridge = Logic::Helpers::Hash.symbolize_keys( - YAML.safe_load_file(file[:path], permitted_classes: [Symbol]) - ).merge({ - system: { - id: file[:path].to_s.sub(/^#{Regexp.escape(file[:base])}/, '').sub(%r{^/}, '').sub(/\.[^.]+\z/, - ''), - path: file[:path], - base: file[:base] - } - }) + cartridge = load_cartridge(file[:path]).merge( + { + system: { + id: file[:path].to_s.sub( + /^#{Regexp.escape(file[:base])}/, '' + ).sub(%r{^/}, '').sub(/\.[^.]+\z/, ''), + path: file[:path], + base: file[:base] + } + } + ) next if cartridge[:meta][:name].nil? diff --git a/controllers/instance.rb b/controllers/instance.rb index 259a548..85b97ec 100644 --- a/controllers/instance.rb +++ b/controllers/instance.rb @@ -1,11 +1,10 @@ # frozen_string_literal: true -require 'yaml' - require_relative '../logic/helpers/hash' require_relative '../components/provider' require_relative '../components/storage' require_relative '../components/stream' +require_relative 'cartridges' require_relative 'interfaces/repl' require_relative 'interfaces/eval' require_relative 'session' @@ -83,13 +82,11 @@ module NanoBot raise StandardError, "Cartridge file not found: \"#{path}\"" end - @cartridge = YAML.safe_load_file(elected_path, permitted_classes: [Symbol]) + @cartridge = Cartridges.load(elected_path) end @safe_cartridge = Marshal.load(Marshal.dump(@cartridge)) - @cartridge = Logic::Helpers::Hash.symbolize_keys(@cartridge) - inject_environment_variables!(@cartridge) end diff --git a/logic/cartridge/parser.rb b/logic/cartridge/parser.rb new file mode 100644 index 0000000..440c929 --- /dev/null +++ b/logic/cartridge/parser.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'singleton' + +require 'redcarpet' +require 'redcarpet/render_strip' + +module NanoBot + module Logic + module Cartridge + module Parser + def self.parse(raw, format:) + normalized = format.to_s.downcase.gsub('.', '') + + if %w[yml yaml].include?(normalized) + yaml(raw) + elsif %w[markdown mdown mkdn md].include?(normalized) + markdown(raw) + else + raise "Unknown cartridge format: '#{format}'" + end + end + + def self.markdown(raw) + yaml(Markdown.instance.render(raw)) + end + + def self.yaml(raw) + Logic::Helpers::Hash.symbolize_keys( + YAML.safe_load(raw, permitted_classes: [Symbol]) + ) + end + + class Renderer < Redcarpet::Render::Base + def block_code(code, _language) + "\n#{code}\n" + end + end + + class Markdown + include Singleton + + attr_reader :markdown + + def initialize + @markdown = Redcarpet::Markdown.new(Renderer, fenced_code_blocks: true) + end + + def render(raw) + @markdown.render(raw) + end + end + end + end + end +end diff --git a/nano-bots.gemspec b/nano-bots.gemspec index f843f37..87c4f48 100644 --- a/nano-bots.gemspec +++ b/nano-bots.gemspec @@ -37,6 +37,7 @@ Gem::Specification.new do |spec| spec.add_dependency 'pry', '~> 0.14.2' spec.add_dependency 'rainbow', '~> 3.1', '>= 3.1.1' spec.add_dependency 'rbnacl', '~> 7.1', '>= 7.1.1' + spec.add_dependency 'redcarpet', '~> 3.6' spec.add_dependency 'sweet-moon', '~> 0.0.7' spec.add_dependency 'cohere-ai', '~> 1.0', '>= 1.0.1' diff --git a/ports/dsl/nano-bots.rb b/ports/dsl/nano-bots.rb index e01b2c4..20d8f14 100644 --- a/ports/dsl/nano-bots.rb +++ b/ports/dsl/nano-bots.rb @@ -8,6 +8,7 @@ require_relative '../../controllers/instance' require_relative '../../controllers/security' require_relative '../../controllers/interfaces/cli' require_relative '../../components/stream' +require_relative 'nano-bots/cartridges' module NanoBot def self.new(cartridge: '-', state: '-', environment: {}) @@ -24,7 +25,7 @@ module NanoBot end def self.cartridges - Controllers::Cartridges.all + Cartridges end def self.cli diff --git a/ports/dsl/nano-bots/cartridges.rb b/ports/dsl/nano-bots/cartridges.rb new file mode 100644 index 0000000..7c0f05b --- /dev/null +++ b/ports/dsl/nano-bots/cartridges.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require_relative '../../controllers/cartridges' + +module NanoBot + module Cartridges + def self.all + Controllers::Cartridges.all + end + + def self.load(path) + Controllers::Cartridges.load(path) + end + end +end diff --git a/spec/data/cartridges/markdown.md b/spec/data/cartridges/markdown.md new file mode 100644 index 0000000..cd50b7b --- /dev/null +++ b/spec/data/cartridges/markdown.md @@ -0,0 +1,37 @@ +A cartridge is a YAML file with human-readable data that outlines the bot's goals, expected behaviors, and settings for authentication and provider utilization. + +We begin with the meta section, which provides information about what this cartridge is designed for: + +```yaml +meta: + symbol: 🤖 + name: ChatGPT 4 Turbo + author: icebaker + version: 0.0.1 + license: CC0-1.0 + description: A helpful assistant. +``` + +It includes details like versioning and license. + +Next, we add a behavior section that will provide the bot with a directive on how it should behave: + +```yaml +behaviors: + interaction: + directive: You are a helpful assistant. +``` + +Now, we need to provide instructions on how this Nano Bot should connect with a provider, which credentials to use, and what specific configurations for the LLM are required: + +```yaml +provider: + id: openai + credentials: + access-token: ENV/OPENAI_API_KEY + settings: + user: ENV/NANO_BOTS_END_USER + model: gpt-4-1106-preview +``` + +In my API, I have set the environment variables `OPENAI_API_KEY` and `NANO_BOTS_END_USER`, which is where the values for these will come from. diff --git a/spec/logic/cartridge/parser_spec.rb b/spec/logic/cartridge/parser_spec.rb new file mode 100644 index 0000000..f8d1302 --- /dev/null +++ b/spec/logic/cartridge/parser_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require_relative '../../../logic/cartridge/parser' + +RSpec.describe NanoBot::Logic::Cartridge::Parser do + context 'markdown' do + let(:raw) { File.read('spec/data/cartridges/markdown.md') } + + it 'parses markdown cartridge' do + expect(described_class.parse(raw, format: 'md')).to eq( + { meta: { + symbol: '🤖', + name: 'ChatGPT 4 Turbo', + author: 'icebaker', + version: '0.0.1', + license: 'CC0-1.0', + description: 'A helpful assistant.' + }, + behaviors: { interaction: { directive: 'You are a helpful assistant.' } }, + provider: { + id: 'openai', + credentials: { 'access-token': 'ENV/OPENAI_API_KEY' }, + settings: { + user: 'ENV/NANO_BOTS_END_USER', + model: 'gpt-4-1106-preview' + } + } } + ) + end + end +end -- cgit v1.2.3 From 4663c3a916a4dfb8e2bf46f6da20b1ad860924e5 Mon Sep 17 00:00:00 2001 From: icebaker Date: Mon, 8 Jan 2024 22:09:37 -0300 Subject: fixing cartridges --- controllers/cartridges.rb | 2 +- logic/cartridge/parser.rb | 6 ++++-- ports/dsl/nano-bots/cartridges.rb | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/controllers/cartridges.rb b/controllers/cartridges.rb index 7215a99..ca1e8f0 100644 --- a/controllers/cartridges.rb +++ b/controllers/cartridges.rb @@ -27,7 +27,7 @@ module NanoBot cartridges = [] files.values.uniq.map do |file| - cartridge = load_cartridge(file[:path]).merge( + cartridge = load(file[:path]).merge( { system: { id: file[:path].to_s.sub( diff --git a/logic/cartridge/parser.rb b/logic/cartridge/parser.rb index 440c929..308ca36 100644 --- a/logic/cartridge/parser.rb +++ b/logic/cartridge/parser.rb @@ -10,7 +10,7 @@ module NanoBot module Cartridge module Parser def self.parse(raw, format:) - normalized = format.to_s.downcase.gsub('.', '') + normalized = format.to_s.downcase.gsub('.', '').strip if %w[yml yaml].include?(normalized) yaml(raw) @@ -32,7 +32,9 @@ module NanoBot end class Renderer < Redcarpet::Render::Base - def block_code(code, _language) + def block_code(code, language) + return nil unless %w[yml yaml].include?(language.to_s.downcase.strip) + "\n#{code}\n" end end diff --git a/ports/dsl/nano-bots/cartridges.rb b/ports/dsl/nano-bots/cartridges.rb index 7c0f05b..40ad14d 100644 --- a/ports/dsl/nano-bots/cartridges.rb +++ b/ports/dsl/nano-bots/cartridges.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative '../../controllers/cartridges' +require_relative '../../../controllers/cartridges' module NanoBot module Cartridges -- cgit v1.2.3 From 309e529759f9a6e9c8538dd5595518897caecfa5 Mon Sep 17 00:00:00 2001 From: icebaker Date: Tue, 9 Jan 2024 21:59:23 -0300 Subject: migrating llama2 to phi-2 --- spec/data/cartridges/models/ollama/llama2.yml | 10 ---------- spec/data/cartridges/models/ollama/phi-2.yml | 10 ++++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) delete mode 100644 spec/data/cartridges/models/ollama/llama2.yml create mode 100644 spec/data/cartridges/models/ollama/phi-2.yml diff --git a/spec/data/cartridges/models/ollama/llama2.yml b/spec/data/cartridges/models/ollama/llama2.yml deleted file mode 100644 index 7f20753..0000000 --- a/spec/data/cartridges/models/ollama/llama2.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -meta: - symbol: 🦙 - name: Llama 2 through Ollama - license: CC0-1.0 - -provider: - id: ollama - settings: - model: llama2 diff --git a/spec/data/cartridges/models/ollama/phi-2.yml b/spec/data/cartridges/models/ollama/phi-2.yml new file mode 100644 index 0000000..5c8e131 --- /dev/null +++ b/spec/data/cartridges/models/ollama/phi-2.yml @@ -0,0 +1,10 @@ +--- +meta: + symbol: 🦙 + name: Phi-2 through Ollama + license: CC0-1.0 + +provider: + id: ollama + settings: + model: phi -- cgit v1.2.3 From 262fd30d28c030cb42b4583c0a6bb394dea12067 Mon Sep 17 00:00:00 2001 From: icebaker Date: Tue, 9 Jan 2024 22:00:00 -0300 Subject: migrating to PATH --- README.md | 8 +++--- components/storage.rb | 44 ++++++++++++++++++++++++++------ controllers/cartridges.rb | 16 ++++++------ ports/dsl/nano-bots/cartridges.rb | 4 +-- spec/components/storage_spec.rb | 53 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 104 insertions(+), 21 deletions(-) create mode 100644 spec/components/storage_spec.rb diff --git a/README.md b/README.md index 7134b28..e1e86d7 100644 --- a/README.md +++ b/README.md @@ -269,8 +269,8 @@ For credentials and configurations, relevant environment variables can be set in export NANO_BOTS_ENCRYPTION_PASSWORD=UNSAFE export NANO_BOTS_END_USER=your-user -# export NANO_BOTS_STATE_DIRECTORY=/home/user/.local/state/nano-bots -# export NANO_BOTS_CARTRIDGES_DIRECTORY=/home/user/.local/share/nano-bots/cartridges +# export NANO_BOTS_STATE_PATH=/home/user/.local/state/nano-bots +# export NANO_BOTS_CARTRIDGES_PATH=/home/user/.local/share/nano-bots/cartridges ``` Alternatively, if your current directory has a `.env` file with the environment variables, they will be automatically loaded: @@ -279,8 +279,8 @@ Alternatively, if your current directory has a `.env` file with the environment NANO_BOTS_ENCRYPTION_PASSWORD=UNSAFE NANO_BOTS_END_USER=your-user -# NANO_BOTS_STATE_DIRECTORY=/home/user/.local/state/nano-bots -# NANO_BOTS_CARTRIDGES_DIRECTORY=/home/user/.local/share/nano-bots/cartridges +# NANO_BOTS_STATE_PATH=/home/user/.local/state/nano-bots +# NANO_BOTS_CARTRIDGES_PATH=/home/user/.local/share/nano-bots/cartridges ``` ### Cohere Command diff --git a/components/storage.rb b/components/storage.rb index 839cd02..270cd81 100644 --- a/components/storage.rb +++ b/components/storage.rb @@ -38,6 +38,7 @@ module NanoBot def self.build_path_and_ensure_state_file!(key, cartridge, environment: {}) path = [ Logic::Helpers::Hash.fetch(cartridge, %i[state directory]), + ENV.fetch('NANO_BOTS_STATE_PATH', nil), ENV.fetch('NANO_BOTS_STATE_DIRECTORY', nil) ].find do |candidate| !candidate.nil? && !candidate.empty? @@ -66,11 +67,31 @@ module NanoBot path end - def self.cartridges_path - [ - ENV.fetch('NANO_BOTS_CARTRIDGES_DIRECTORY', nil), - "#{user_home!.sub(%r{/$}, '')}/.local/share/nano-bots/cartridges" - ].compact.uniq.filter { |path| File.directory?(path) }.compact.first + def self.cartridges_path(components: {}) + components[:directory?] = ->(path) { File.directory?(path) } unless components.key?(:directory?) + components[:ENV] = ENV unless components.key?(:ENV) + + default = "#{user_home!(components:).sub(%r{/$}, '')}/.local/share/nano-bots/cartridges" + + from_environment = [ + components[:ENV].fetch('NANO_BOTS_CARTRIDGES_PATH', nil), + components[:ENV].fetch('NANO_BOTS_CARTRIDGES_DIRECTORY', nil) + ].compact + + elected = [ + from_environment.empty? ? nil : from_environment.join(':'), + default + ].compact.uniq.filter do |path| + path.split(':').any? { |candidate| components[:directory?].call(candidate) } + end.compact.first + + return default unless elected + + elected = elected.split(':').filter do |path| + components[:directory?].call(path) + end.compact + + elected.size.positive? ? elected.join(':') : default end def self.cartridge_path(path) @@ -82,9 +103,14 @@ module NanoBot candidates << "#{partial}.#{extension}" end - unless ENV.fetch('NANO_BOTS_CARTRIDGES_DIRECTORY', nil).nil? - directory = ENV.fetch('NANO_BOTS_CARTRIDGES_DIRECTORY').sub(%r{/$}, '') + directories = [ + ENV.fetch('NANO_BOTS_CARTRIDGES_PATH', nil), + ENV.fetch('NANO_BOTS_CARTRIDGES_DIRECTORY', nil) + ].compact.map do |directory| + directory.split(':') + end.flatten.map { |directory| directory.sub(%r{/$}, '') } + directories.each do |directory| partial = File.join(File.dirname(partial), File.basename(partial, File.extname(partial))) partial = partial.sub(%r{^\.?/}, '') @@ -115,7 +141,9 @@ module NanoBot end end - def self.user_home! + def self.user_home!(components: {}) + return components[:home] if components[:home] + [Dir.home, `echo ~`.strip, '~'].find do |candidate| !candidate.nil? && !candidate.empty? end diff --git a/controllers/cartridges.rb b/controllers/cartridges.rb index ca1e8f0..3be8d53 100644 --- a/controllers/cartridges.rb +++ b/controllers/cartridges.rb @@ -12,16 +12,18 @@ module NanoBot Logic::Cartridge::Parser.parse(File.read(path), format: File.extname(path)) end - def self.all + def self.all(components: {}) files = {} - path = Components::Storage.cartridges_path + paths = Components::Storage.cartridges_path(components:) - Dir.glob("#{path}/**/*.{yml,yaml,markdown,mdown,mkdn,md}").each do |file| - files[Pathname.new(file).realpath] = { - base: path, - path: Pathname.new(file).realpath - } + paths.split(':').each do |path| + Dir.glob("#{path}/**/*.{yml,yaml,markdown,mdown,mkdn,md}").each do |file| + files[Pathname.new(file).realpath] = { + base: path, + path: Pathname.new(file).realpath + } + end end cartridges = [] diff --git a/ports/dsl/nano-bots/cartridges.rb b/ports/dsl/nano-bots/cartridges.rb index 40ad14d..fb23c39 100644 --- a/ports/dsl/nano-bots/cartridges.rb +++ b/ports/dsl/nano-bots/cartridges.rb @@ -4,8 +4,8 @@ require_relative '../../../controllers/cartridges' module NanoBot module Cartridges - def self.all - Controllers::Cartridges.all + def self.all(components: {}) + Controllers::Cartridges.all(components:) end def self.load(path) diff --git a/spec/components/storage_spec.rb b/spec/components/storage_spec.rb new file mode 100644 index 0000000..99131dd --- /dev/null +++ b/spec/components/storage_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require_relative '../../components/storage' + +RSpec.describe NanoBot::Components::Storage do + it 'symbolizes keys' do + expect( + described_class.cartridges_path( + components: { home: '/home/aqua', ENV: {}, directory?: ->(_) { true } } + ) + ).to eq('/home/aqua/.local/share/nano-bots/cartridges') + + expect( + described_class.cartridges_path( + components: { + home: '/home/aqua', + ENV: { 'NANO_BOTS_CARTRIDGES_DIRECTORY' => '/home/aqua/my-cartridges' }, + directory?: ->(_) { true } + } + ) + ).to eq('/home/aqua/my-cartridges') + + expect( + described_class.cartridges_path( + components: { + home: '/home/aqua', + ENV: { + 'NANO_BOTS_CARTRIDGES_DIRECTORY' => '/home/aqua/my-cartridges', + 'NANO_BOTS_CARTRIDGES_PATH' => '/home/aqua/lime/my-cartridges' + }, + directory?: ->(_) { true } + } + ) + ).to eq('/home/aqua/lime/my-cartridges:/home/aqua/my-cartridges') + + expect( + described_class.cartridges_path( + components: { + home: '/home/aqua', + ENV: { + 'NANO_BOTS_CARTRIDGES_DIRECTORY' => '/home/aqua/my-cartridges', + 'NANO_BOTS_CARTRIDGES_PATH' => '/home/aqua/lime/my-cartridges:/home/aqua/ivory/my-cartridges' + }, + directory?: lambda do |path| + { '/home/aqua/my-cartridges' => true, + '/home/aqua/lime/my-cartridge' => false, + '/home/aqua/ivory/my-cartridges' => true }[path] + end + } + ) + ).to eq('/home/aqua/ivory/my-cartridges:/home/aqua/my-cartridges') + end +end -- cgit v1.2.3 From a36604d266cb019311cdf67110596a16986a0fff Mon Sep 17 00:00:00 2001 From: icebaker Date: Tue, 9 Jan 2024 22:15:08 -0300 Subject: migrating to PATH --- README.md | 2 +- components/storage.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e1e86d7..61a35b7 100644 --- a/README.md +++ b/README.md @@ -963,7 +963,7 @@ cd ruby-nano-bots cp docker-compose.example.yml docker-compose.yml ``` -Set your provider credentials and choose your desired directory for the cartridges files: +Set your provider credentials and choose your desired path for the cartridges files: ### Cohere Command Container diff --git a/components/storage.rb b/components/storage.rb index 270cd81..10ea335 100644 --- a/components/storage.rb +++ b/components/storage.rb @@ -37,6 +37,7 @@ module NanoBot def self.build_path_and_ensure_state_file!(key, cartridge, environment: {}) path = [ + Logic::Helpers::Hash.fetch(cartridge, %i[state path]), Logic::Helpers::Hash.fetch(cartridge, %i[state directory]), ENV.fetch('NANO_BOTS_STATE_PATH', nil), ENV.fetch('NANO_BOTS_STATE_DIRECTORY', nil) -- cgit v1.2.3 From 3dc22548895718ffc7396227267ecbb4902b62f9 Mon Sep 17 00:00:00 2001 From: icebaker Date: Wed, 10 Jan 2024 20:05:48 -0300 Subject: improving markdown parser --- components/embedding.rb | 2 +- components/providers/openai.rb | 4 +- logic/cartridge/parser.rb | 84 +++++++++++++++++++++++++--- logic/helpers/hash.rb | 13 +++++ spec/data/cartridges/block.md | 7 +++ spec/data/cartridges/tools.md | 76 ++++++++++++++++++++++++++ spec/logic/cartridge/parser_spec.rb | 106 +++++++++++++++++++++++++++++------- spec/logic/helpers/hash_spec.rb | 9 +++ 8 files changed, 270 insertions(+), 31 deletions(-) create mode 100644 spec/data/cartridges/block.md create mode 100644 spec/data/cartridges/tools.md diff --git a/components/embedding.rb b/components/embedding.rb index f08ff3d..4dd2e6c 100644 --- a/components/embedding.rb +++ b/components/embedding.rb @@ -45,7 +45,7 @@ module NanoBot def self.clojure(source:, parameters:, values:, safety:) ensure_safety!(safety) - raise 'TODO: sandboxed Clojure through Babashka not implemented' if safety[:sandboxed] + raise 'Sandboxed Clojure not supported.' if safety[:sandboxed] raise 'invalid Clojure parameter name' if parameters.include?('injected-parameters') diff --git a/components/providers/openai.rb b/components/providers/openai.rb index e71f143..79b935e 100644 --- a/components/providers/openai.rb +++ b/components/providers/openai.rb @@ -140,7 +140,7 @@ module NanoBot begin @client.chat(parameters: Logic::OpenAI::Tokens.apply_policies!(cartridge, payload)) rescue StandardError => e - raise e.class, e.response[:body] if e.response && e.response[:body] + raise e.class, e.response[:body] if e.respond_to?(:response) && e.response && e.response[:body] raise e end @@ -148,7 +148,7 @@ module NanoBot begin result = @client.chat(parameters: Logic::OpenAI::Tokens.apply_policies!(cartridge, payload)) rescue StandardError => e - raise e.class, e.response[:body] if e.response && e.response[:body] + raise e.class, e.response[:body] if e.respond_to?(:response) && e.response && e.response[:body] raise e end diff --git a/logic/cartridge/parser.rb b/logic/cartridge/parser.rb index 308ca36..f82b968 100644 --- a/logic/cartridge/parser.rb +++ b/logic/cartridge/parser.rb @@ -22,7 +22,50 @@ module NanoBot end def self.markdown(raw) - yaml(Markdown.instance.render(raw)) + yaml_source = [] + + tools = [] + + blocks = Markdown.new.render(raw).blocks + + previous_block_is_tool = false + + blocks.each do |block| + if block[:language] == 'yaml' + parsed = Logic::Helpers::Hash.symbolize_keys( + YAML.safe_load(block[:source], permitted_classes: [Symbol]) + ) + + if parsed.key?(:tools) && parsed[:tools].is_a?(Array) && !parsed[:tools].empty? + previous_block_is_tool = true + + tools.concat(parsed[:tools]) + + parsed.delete(:tools) + + unless parsed.empty? + yaml_source << YAML.dump(Logic::Helpers::Hash.stringify_keys( + parsed + )).gsub(/^---/, '') # TODO: Is this safe enough? + end + else + yaml_source << block[:source] + previous_block_is_tool = false + nil + end + elsif previous_block_is_tool + tools.last[block[:language].to_sym] = block[:source] + previous_block_is_tool = false + end + end + + unless tools.empty? + yaml_source << YAML.dump(Logic::Helpers::Hash.stringify_keys( + { tools: } + )).gsub(/^---/, '') # TODO: Is this safe enough? + end + + yaml(yaml_source.join("\n")) end def self.yaml(raw) @@ -32,24 +75,51 @@ module NanoBot end class Renderer < Redcarpet::Render::Base + LANGUAGES_MAP = { + 'yml' => 'yaml', + 'yaml' => 'yaml', + 'lua' => 'lua', + 'fnl' => 'fennel', + 'fennel' => 'fennel', + 'clj' => 'clojure', + 'clojure' => 'clojure' + }.freeze + + LANGUAGES = LANGUAGES_MAP.keys.freeze + + def initialize(...) + super(...) + @_nano_bots_blocks = [] + end + + attr_reader :_nano_bots_blocks + def block_code(code, language) - return nil unless %w[yml yaml].include?(language.to_s.downcase.strip) + key = language.to_s.downcase.strip - "\n#{code}\n" + return nil unless LANGUAGES.include?(key) + + @_nano_bots_blocks << { language: LANGUAGES_MAP[key], source: code } + + nil end end class Markdown - include Singleton - attr_reader :markdown def initialize - @markdown = Redcarpet::Markdown.new(Renderer, fenced_code_blocks: true) + @renderer = Renderer.new + @markdown = Redcarpet::Markdown.new(@renderer, fenced_code_blocks: true) + end + + def blocks + @renderer._nano_bots_blocks end def render(raw) - @markdown.render(raw) + @markdown.render(raw.gsub(/```\w/, "\n\n\\0")) + self end end end diff --git a/logic/helpers/hash.rb b/logic/helpers/hash.rb index 90432b5..4cb44ac 100644 --- a/logic/helpers/hash.rb +++ b/logic/helpers/hash.rb @@ -17,6 +17,19 @@ module NanoBot end end + def self.stringify_keys(object) + case object + when ::Hash + object.each_with_object({}) do |(key, value), result| + result[key.to_s] = stringify_keys(value) + end + when Array + object.map { |e| stringify_keys(e) } + else + object + end + end + def self.fetch(object, path) node = object diff --git a/spec/data/cartridges/block.md b/spec/data/cartridges/block.md new file mode 100644 index 0000000..ef8588d --- /dev/null +++ b/spec/data/cartridges/block.md @@ -0,0 +1,7 @@ +First, we need to add some important details: +```yaml +safety: + functions: + sandboxed: false +``` +Hi! diff --git a/spec/data/cartridges/tools.md b/spec/data/cartridges/tools.md new file mode 100644 index 0000000..5d2da5a --- /dev/null +++ b/spec/data/cartridges/tools.md @@ -0,0 +1,76 @@ +A cartridge is a YAML file with human-readable data that outlines the bot's goals, expected behaviors, and settings for authentication and provider utilization. + +We begin with the meta section, which provides information about what this cartridge is designed for: + +```yaml +meta: + symbol: 🕛 + name: Date and Time + author: icebaker + version: 0.0.1 + license: CC0-1.0 + description: A helpful assistant. +``` + +It includes details like versioning and license. + +Next, we add a behavior section that will provide the bot with a directive on how it should behave: + +```yaml +behaviors: + interaction: + directive: You are a helpful assistant. +``` + +Now, we need to provide instructions on how this Nano Bot should connect with a provider, which credentials to use, and what specific configurations for the LLM are required: + +```yaml +provider: + id: openai + credentials: + access-token: ENV/OPENAI_API_KEY + settings: + user: ENV/NANO_BOTS_END_USER + model: gpt-4-1106-preview +``` + +In my API, I have set the environment variables `OPENAI_API_KEY` and `NANO_BOTS_END_USER`, which is where the values for these will come from. + +Nano Bot ready; let's start adding some extra power to it. + +## Random Numbers + +```yml +tools: +- name: random-number + description: Generates a random number within a given range. + parameters: + type: object + properties: + from: + type: integer + description: The minimum expected number for random generation. + to: + type: integer + description: The maximum expected number for random generation. + required: + - from + - to +``` + +```clj +(let [{:strs [from to]} parameters] + (+ from (rand-int (+ 1 (- to from))))) +``` + +## Date and Time + +```yaml +tools: +- name: date-and-time + description: Returns the current date and time. +``` + +```fnl +(os.date) +``` diff --git a/spec/logic/cartridge/parser_spec.rb b/spec/logic/cartridge/parser_spec.rb index f8d1302..e8bac0b 100644 --- a/spec/logic/cartridge/parser_spec.rb +++ b/spec/logic/cartridge/parser_spec.rb @@ -4,28 +4,92 @@ require_relative '../../../logic/cartridge/parser' RSpec.describe NanoBot::Logic::Cartridge::Parser do context 'markdown' do - let(:raw) { File.read('spec/data/cartridges/markdown.md') } + context 'default' do + let(:raw) { File.read('spec/data/cartridges/markdown.md') } - it 'parses markdown cartridge' do - expect(described_class.parse(raw, format: 'md')).to eq( - { meta: { - symbol: '🤖', - name: 'ChatGPT 4 Turbo', - author: 'icebaker', - version: '0.0.1', - license: 'CC0-1.0', - description: 'A helpful assistant.' - }, - behaviors: { interaction: { directive: 'You are a helpful assistant.' } }, - provider: { - id: 'openai', - credentials: { 'access-token': 'ENV/OPENAI_API_KEY' }, - settings: { - user: 'ENV/NANO_BOTS_END_USER', - model: 'gpt-4-1106-preview' - } - } } - ) + it 'parses markdown cartridge' do + expect(described_class.parse(raw, format: 'md')).to eq( + { meta: { + symbol: '🤖', + name: 'ChatGPT 4 Turbo', + author: 'icebaker', + version: '0.0.1', + license: 'CC0-1.0', + description: 'A helpful assistant.' + }, + behaviors: { interaction: { directive: 'You are a helpful assistant.' } }, + provider: { + id: 'openai', + credentials: { 'access-token': 'ENV/OPENAI_API_KEY' }, + settings: { + user: 'ENV/NANO_BOTS_END_USER', + model: 'gpt-4-1106-preview' + } + } } + ) + end + end + + context 'tools' do + let(:raw) { File.read('spec/data/cartridges/tools.md') } + + it 'parses markdown cartridge' do + expect(described_class.parse(raw, format: 'md')).to eq( + { meta: { + symbol: '🕛', + name: 'Date and Time', + author: 'icebaker', + version: '0.0.1', + license: 'CC0-1.0', + description: 'A helpful assistant.' + }, + behaviors: { + interaction: { + directive: 'You are a helpful assistant.' + } + }, + provider: { + id: 'openai', + credentials: { 'access-token': 'ENV/OPENAI_API_KEY' }, + settings: { + user: 'ENV/NANO_BOTS_END_USER', + model: 'gpt-4-1106-preview' + } + }, + tools: [ + { name: 'random-number', + description: 'Generates a random number within a given range.', + parameters: { + type: 'object', + properties: { + from: { + type: 'integer', + description: 'The minimum expected number for random generation.' + }, + to: { + type: 'integer', + description: 'The maximum expected number for random generation.' + } + }, + required: %w[from to] + }, + clojure: "(let [{:strs [from to]} parameters]\n (+ from (rand-int (+ 1 (- to from)))))\n" }, + { name: 'date-and-time', + description: 'Returns the current date and time.', + fennel: "(os.date)\n" } + ] } + ) + end + end + + context 'block' do + let(:raw) { File.read('spec/data/cartridges/block.md') } + + it 'parses markdown cartridge' do + expect(described_class.parse(raw, format: 'md')).to eq( + { safety: { functions: { sandboxed: false } } } + ) + end end end end diff --git a/spec/logic/helpers/hash_spec.rb b/spec/logic/helpers/hash_spec.rb index 09012c8..5e4ec60 100644 --- a/spec/logic/helpers/hash_spec.rb +++ b/spec/logic/helpers/hash_spec.rb @@ -7,7 +7,16 @@ RSpec.describe NanoBot::Logic::Helpers::Hash do expect(described_class.symbolize_keys({ 'a' => 'b', 'c' => { 'd' => ['e'] } })).to eq( { a: 'b', c: { d: ['e'] } } ) + end + + it 'stringify keys' do + pp described_class.stringify_keys({ a: 'b', c: { d: [:e] } }) + expect(described_class.stringify_keys({ a: 'b', c: { d: [:e] } })).to eq( + { 'a' => 'b', 'c' => { 'd' => [:e] } } + ) + end + it 'fetch a path of keys' do expect(described_class.fetch({ a: 'b', c: { d: ['e'] } }, %i[c d])).to eq( ['e'] ) -- cgit v1.2.3 From 552c9c7321895d41f81e460b94182b7a00d2cb88 Mon Sep 17 00:00:00 2001 From: icebaker Date: Wed, 10 Jan 2024 20:23:06 -0300 Subject: fixing spec --- spec/logic/helpers/hash_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/logic/helpers/hash_spec.rb b/spec/logic/helpers/hash_spec.rb index 5e4ec60..7c8ff58 100644 --- a/spec/logic/helpers/hash_spec.rb +++ b/spec/logic/helpers/hash_spec.rb @@ -10,7 +10,6 @@ RSpec.describe NanoBot::Logic::Helpers::Hash do end it 'stringify keys' do - pp described_class.stringify_keys({ a: 'b', c: { d: [:e] } }) expect(described_class.stringify_keys({ a: 'b', c: { d: [:e] } })).to eq( { 'a' => 'b', 'c' => { 'd' => [:e] } } ) -- cgit v1.2.3 From aabf3d9b711f66fe4195a8c850856826c7ad5580 Mon Sep 17 00:00:00 2001 From: icebaker Date: Wed, 10 Jan 2024 21:31:42 -0300 Subject: improving merging approach --- logic/cartridge/parser.rb | 20 +++++++++++++------- logic/helpers/hash.rb | 10 ++++++++++ spec/data/cartridges/meta.md | 25 +++++++++++++++++++++++++ spec/logic/cartridge/parser_spec.rb | 24 ++++++++++++++++++++++++ spec/logic/helpers/hash_spec.rb | 9 +++++++++ 5 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 spec/data/cartridges/meta.md diff --git a/logic/cartridge/parser.rb b/logic/cartridge/parser.rb index f82b968..50b3dc5 100644 --- a/logic/cartridge/parser.rb +++ b/logic/cartridge/parser.rb @@ -44,9 +44,9 @@ module NanoBot parsed.delete(:tools) unless parsed.empty? - yaml_source << YAML.dump(Logic::Helpers::Hash.stringify_keys( - parsed - )).gsub(/^---/, '') # TODO: Is this safe enough? + yaml_source << YAML.dump( + Logic::Helpers::Hash.stringify_keys(parsed) + ).gsub(/^---/, '') # TODO: Is this safe enough? end else yaml_source << block[:source] @@ -60,12 +60,18 @@ module NanoBot end unless tools.empty? - yaml_source << YAML.dump(Logic::Helpers::Hash.stringify_keys( - { tools: } - )).gsub(/^---/, '') # TODO: Is this safe enough? + yaml_source << YAML.dump( + Logic::Helpers::Hash.stringify_keys({ tools: }) + ).gsub(/^---/, '') # TODO: Is this safe enough? end - yaml(yaml_source.join("\n")) + cartridge = {} + + yaml_source.each do |source| + cartridge = Logic::Helpers::Hash.deep_merge(cartridge, yaml(source)) + end + + cartridge end def self.yaml(raw) diff --git a/logic/helpers/hash.rb b/logic/helpers/hash.rb index 4cb44ac..66b6742 100644 --- a/logic/helpers/hash.rb +++ b/logic/helpers/hash.rb @@ -4,6 +4,16 @@ module NanoBot module Logic module Helpers module Hash + def self.deep_merge(hash1, hash2) + hash1.merge(hash2) do |_key, old_val, new_val| + if old_val.is_a?(::Hash) && new_val.is_a?(::Hash) + deep_merge(old_val, new_val) + else + new_val + end + end + end + def self.symbolize_keys(object) case object when ::Hash diff --git a/spec/data/cartridges/meta.md b/spec/data/cartridges/meta.md new file mode 100644 index 0000000..68a0cbd --- /dev/null +++ b/spec/data/cartridges/meta.md @@ -0,0 +1,25 @@ +Start by defining a meta section: + +```yaml +meta: + symbol: 🤖 + name: Nano Bot Name + author: Your Name + description: A helpful assistant. +``` + +You can also add version and license information: + +```yaml +meta: + version: 1.0.0 + license: CC0-1.0 +``` + +Then, add a behavior section: + +```yaml +behaviors: + interaction: + directive: You are a helpful assistant. +``` diff --git a/spec/logic/cartridge/parser_spec.rb b/spec/logic/cartridge/parser_spec.rb index e8bac0b..8297baa 100644 --- a/spec/logic/cartridge/parser_spec.rb +++ b/spec/logic/cartridge/parser_spec.rb @@ -30,6 +30,30 @@ RSpec.describe NanoBot::Logic::Cartridge::Parser do end end + context 'meta' do + let(:raw) { File.read('spec/data/cartridges/meta.md') } + + it 'parses markdown cartridge' do + expect(described_class.parse(raw, format: 'md')).to eq( + { + meta: { + symbol: '🤖', + name: 'Nano Bot Name', + author: 'Your Name', + description: 'A helpful assistant.', + version: '1.0.0', + license: 'CC0-1.0' + }, + behaviors: { + interaction: { + directive: 'You are a helpful assistant.' + } + } + } + ) + end + end + context 'tools' do let(:raw) { File.read('spec/data/cartridges/tools.md') } diff --git a/spec/logic/helpers/hash_spec.rb b/spec/logic/helpers/hash_spec.rb index 7c8ff58..0da92fb 100644 --- a/spec/logic/helpers/hash_spec.rb +++ b/spec/logic/helpers/hash_spec.rb @@ -9,6 +9,15 @@ RSpec.describe NanoBot::Logic::Helpers::Hash do ) end + it 'deep merges' do + expect(described_class.deep_merge( + { a: { x: 1, y: 2 }, b: 3 }, + { a: { y: 99, z: 4 }, c: 5 } + )).to eq( + { a: { x: 1, y: 99, z: 4 }, b: 3, c: 5 } + ) + end + it 'stringify keys' do expect(described_class.stringify_keys({ a: 'b', c: { d: [:e] } })).to eq( { 'a' => 'b', 'c' => { 'd' => [:e] } } -- cgit v1.2.3