Christoph Marzell
5 days ago
14 changed files with 336 additions and 3 deletions
@ -0,0 +1,14 @@ |
|||||
|
# app/jobs/training_watch_job.rb |
||||
|
class TrainingWatchJob < ApplicationJob |
||||
|
queue_as :default |
||||
|
|
||||
|
def perform |
||||
|
results = TrainingWatch::Checker.new.run! |
||||
|
return if results.empty? |
||||
|
|
||||
|
email = ENV.fetch("TRAINING_WATCH_NOTIFY_EMAIL", "christoph@marzell.net") |
||||
|
return if email.blank? |
||||
|
|
||||
|
TrainingWatchMailer.notify(email: email, results: results).deliver_now |
||||
|
end |
||||
|
end |
||||
@ -0,0 +1,7 @@ |
|||||
|
# app/mailers/training_watch_mailer.rb |
||||
|
class TrainingWatchMailer < ApplicationMailer |
||||
|
def notify(email:, results:) |
||||
|
@results = results |
||||
|
mail(to: email, subject: "Lehrgänge 2027: neue Treffer gefunden") |
||||
|
end |
||||
|
end |
||||
@ -0,0 +1,7 @@ |
|||||
|
# frozen_string_literal: true |
||||
|
# app/models/training_watch_hit.rb |
||||
|
class TrainingWatchHit < ApplicationRecord |
||||
|
belongs_to :training_watch_source |
||||
|
|
||||
|
validates :fingerprint, presence: true, uniqueness: true |
||||
|
end |
||||
@ -0,0 +1,10 @@ |
|||||
|
# frozen_string_literal: true |
||||
|
|
||||
|
# app/models/training_watch_source.rb |
||||
|
class TrainingWatchSource < ApplicationRecord |
||||
|
has_many :training_watch_hits, dependent: :delete_all |
||||
|
|
||||
|
validates :name, :url, :kind, presence: true |
||||
|
validates :url, uniqueness: true |
||||
|
validates :kind, inclusion: { in: %w[html rss] } |
||||
|
end |
||||
@ -0,0 +1,183 @@ |
|||||
|
# app/services/training_watch/checker.rb |
||||
|
require "digest" |
||||
|
require "faraday" |
||||
|
require "nokogiri" |
||||
|
|
||||
|
module TrainingWatch |
||||
|
class Checker |
||||
|
DEFAULT_REGEX = / |
||||
|
\b2027\b |
||||
|
.*? |
||||
|
(start|beginn|lehrgang|curriculum|gruppe|kurs|aufnahme|bewerb|anmeld|termin) |
||||
|
/imx |
||||
|
|
||||
|
USER_AGENT = "RailsTrainingWatch/1.0 (+https://your-app.example)" |
||||
|
|
||||
|
Result = Struct.new(:source, :new_hits, keyword_init: true) |
||||
|
|
||||
|
def initialize(sources: TrainingWatchSource.where(enabled: true), logger: Rails.logger) |
||||
|
@sources = sources |
||||
|
@logger = logger |
||||
|
@http = Faraday.new do |f| |
||||
|
f.options.timeout = 15 |
||||
|
f.options.open_timeout = 10 |
||||
|
f.headers["User-Agent"] = USER_AGENT |
||||
|
f.adapter Faraday.default_adapter |
||||
|
end |
||||
|
end |
||||
|
|
||||
|
def run! |
||||
|
results = [] |
||||
|
|
||||
|
@sources.find_each do |source| |
||||
|
begin |
||||
|
new_hits = check_one!(source) |
||||
|
results << Result.new(source:, new_hits:) if new_hits.any? |
||||
|
rescue => e |
||||
|
@logger.warn("[TrainingWatch] source=#{source.id} #{source.name} failed: #{e.class}: #{e.message}") |
||||
|
ensure |
||||
|
source.update!(last_checked_at: Time.current) |
||||
|
end |
||||
|
end |
||||
|
|
||||
|
results |
||||
|
end |
||||
|
|
||||
|
private |
||||
|
|
||||
|
def check_one!(source) |
||||
|
response = fetch(source) |
||||
|
return [] if response.nil? |
||||
|
|
||||
|
case source.kind |
||||
|
when "html" then check_html(source, response) |
||||
|
when "rss" then check_rss(source, response) # optional, stub unten |
||||
|
else [] |
||||
|
end |
||||
|
end |
||||
|
|
||||
|
def fetch(source) |
||||
|
headers = {} |
||||
|
headers["If-None-Match"] = source.last_etag if source.last_etag.present? |
||||
|
headers["If-Modified-Since"] = source.last_modified if source.last_modified.present? |
||||
|
|
||||
|
resp = @http.get(source.url, nil, headers) |
||||
|
|
||||
|
if resp.status == 304 |
||||
|
@logger.info("[TrainingWatch] source=#{source.id} not modified") |
||||
|
return nil |
||||
|
end |
||||
|
|
||||
|
if resp.status >= 400 |
||||
|
raise "HTTP #{resp.status}" |
||||
|
end |
||||
|
|
||||
|
source.update!( |
||||
|
last_etag: resp.headers["etag"], |
||||
|
last_modified: resp.headers["last-modified"] |
||||
|
) |
||||
|
|
||||
|
resp |
||||
|
end |
||||
|
|
||||
|
def check_html(source, response) |
||||
|
html = response.body.to_s |
||||
|
doc = Nokogiri::HTML(html) |
||||
|
|
||||
|
page_title = doc.at_css("title")&.text&.strip |
||||
|
text = doc.text.gsub(/\s+/, " ").strip |
||||
|
|
||||
|
regex = compile_regex(source) |
||||
|
return [] unless text.match?(regex) |
||||
|
|
||||
|
# Links sammeln, die auch matchen könnten (oder zumindest "2027" enthalten) |
||||
|
candidates = doc.css("a").map do |a| |
||||
|
href = a["href"].to_s.strip |
||||
|
next if href.empty? |
||||
|
|
||||
|
label = a.text.to_s.gsub(/\s+/, " ").strip |
||||
|
abs = absolutize_url(source.url, href) |
||||
|
{ label:, url: abs } |
||||
|
end.compact |
||||
|
|
||||
|
# Wenn Links explizit 2027 enthalten, priorisieren, sonst Seitenhit |
||||
|
link_hits = candidates.select { |c| (c[:label] + " " + c[:url]).match?(/\b2027\b/i) }.first(10) |
||||
|
|
||||
|
hits = [] |
||||
|
if link_hits.any? |
||||
|
link_hits.each do |c| |
||||
|
hits << create_hit!( |
||||
|
source:, |
||||
|
title: c[:label].presence || page_title, |
||||
|
hit_url: c[:url], |
||||
|
snippet: build_snippet(text, regex) |
||||
|
) |
||||
|
end |
||||
|
else |
||||
|
hits << create_hit!( |
||||
|
source:, |
||||
|
title: page_title || source.name, |
||||
|
hit_url: source.url, |
||||
|
snippet: build_snippet(text, regex) |
||||
|
) |
||||
|
end |
||||
|
|
||||
|
hits.compact |
||||
|
end |
||||
|
|
||||
|
# Optional: RSS support (wenn du später Feed-URLs nutzt) |
||||
|
def check_rss(source, response) |
||||
|
# Minimal: RSS als Text matchen (robust genug für "2027") |
||||
|
body = response.body.to_s |
||||
|
regex = compile_regex(source) |
||||
|
return [] unless body.match?(regex) |
||||
|
|
||||
|
create = create_hit!( |
||||
|
source:, |
||||
|
title: source.name, |
||||
|
hit_url: source.url, |
||||
|
snippet: build_snippet(body.gsub(/\s+/, " "), regex) |
||||
|
) |
||||
|
create ? [create] : [] |
||||
|
end |
||||
|
|
||||
|
def compile_regex(source) |
||||
|
return DEFAULT_REGEX if source.match_regex.blank? |
||||
|
Regexp.new(source.match_regex, Regexp::IGNORECASE | Regexp::MULTILINE) |
||||
|
rescue RegexpError |
||||
|
DEFAULT_REGEX |
||||
|
end |
||||
|
|
||||
|
def build_snippet(text, regex) |
||||
|
m = text.match(regex) |
||||
|
return text[0, 280] if m.nil? |
||||
|
|
||||
|
start = [m.begin(0) - 120, 0].max |
||||
|
slice = text[start, 380] || text |
||||
|
slice.strip |
||||
|
end |
||||
|
|
||||
|
def absolutize_url(base, href) |
||||
|
uri = URI.parse(href) |
||||
|
return uri.to_s if uri.absolute? |
||||
|
URI.join(base, href).to_s |
||||
|
rescue |
||||
|
href |
||||
|
end |
||||
|
|
||||
|
def create_hit!(source:, title:, hit_url:, snippet:, published_at: nil) |
||||
|
fp = Digest::SHA256.hexdigest([source.id, title.to_s, hit_url.to_s, snippet.to_s[0, 200]].join("|")) |
||||
|
|
||||
|
TrainingWatchHit.create!( |
||||
|
training_watch_source: source, |
||||
|
title: title.to_s.strip.presence, |
||||
|
hit_url: hit_url.to_s.strip.presence, |
||||
|
snippet: snippet.to_s.strip.presence, |
||||
|
published_at:, |
||||
|
fingerprint: fp |
||||
|
) |
||||
|
rescue ActiveRecord::RecordNotUnique |
||||
|
nil |
||||
|
end |
||||
|
end |
||||
|
end |
||||
@ -0,0 +1,12 @@ |
|||||
|
<!-- app/views/training_watch_mailer/notify.text.erb --> |
||||
|
Neue Treffer für Lehrgänge Start 2027: |
||||
|
|
||||
|
<% @results.each do |r| %> |
||||
|
Quelle: <%= r.source.name %> (<%= r.source.url %>) |
||||
|
<% r.new_hits.each do |h| %> |
||||
|
- <%= h.title.presence || "Treffer" %> |
||||
|
<%= h.hit_url %> |
||||
|
<%= h.snippet %> |
||||
|
<% end %> |
||||
|
|
||||
|
<% end %> |
||||
@ -0,0 +1,18 @@ |
|||||
|
class CreateTrainingWatchSources < ActiveRecord::Migration[7.1] |
||||
|
def change |
||||
|
create_table :training_watch_sources do |t| |
||||
|
t.string :name, null: false |
||||
|
t.string :url, null: false |
||||
|
t.string :kind, null: false, default: "html" # html | rss (optional) |
||||
|
t.string :match_regex # optional: überschreibt Default-Matcher |
||||
|
t.string :last_etag |
||||
|
t.string :last_modified |
||||
|
t.datetime :last_checked_at |
||||
|
t.boolean :enabled, null: false, default: true |
||||
|
t.timestamps |
||||
|
end |
||||
|
|
||||
|
add_index :training_watch_sources, :enabled |
||||
|
add_index :training_watch_sources, :url, unique: true |
||||
|
end |
||||
|
end |
||||
@ -0,0 +1,16 @@ |
|||||
|
class CreateTrainingWatchHits < ActiveRecord::Migration[7.1] |
||||
|
def change |
||||
|
create_table :training_watch_hits do |t| |
||||
|
t.references :training_watch_source, null: false, foreign_key: true |
||||
|
t.string :title |
||||
|
t.string :hit_url |
||||
|
t.datetime :published_at |
||||
|
t.text :snippet |
||||
|
t.string :fingerprint, null: false |
||||
|
t.timestamps |
||||
|
end |
||||
|
|
||||
|
add_index :training_watch_hits, :fingerprint, unique: true |
||||
|
add_index :training_watch_hits, :created_at |
||||
|
end |
||||
|
end |
||||
Loading…
Reference in new issue