로그스태시로 데이터베이스에서 레코드 가져오기

mysql

input {
  jdbc {
    jdbc_driver_library => "/path/to/your/jdbc/driver.jar"
    
    # Use the appropriate driver for your database (e.g., MySQL, Oracle)
    jdbc_driver_class => "org.postgresql.Driver"  
    jdbc_connection_string => "jdbc:postgresql://hostname:port/dbname"  # Connection URL
    jdbc_user => "your_db_user"
    jdbc_password => "your_db_password"
    statement => "SELECT * FROM your_table"  # SQL query to fetch the records
    
    # How often to run the query (this runs the query every minute)
    schedule => "* * * * *"  
    
    # Cron expression to run every day at midnight
    # Run the query once a day 
    schedule => "0 0 0 * * *" 
    
    #How many rows to fetch at once from MySQL 
    #set this to 5000 or higher to fetch large batches
    jdbc_fetch_size => 5000
    jdbc_page_size => 5000
    
    use_column_value => true
     # Column used for keeping trtracking_columnack of records fetched in subsequent runs (e.g., primary key)
    tracking_column => "id" 
    tracking_column_type => "numeric"  # or "date" based on your tracking column
    last_run_metadata_path => "/path/to/logstash/.last_run"  # Path to save the last run timestamp or id
  }
}

 

 

jdbc_fetch_size => 5000 

jdbc_paging_enabled => true
jdbc_page_size => 10000

 

Run Logstash in debug mode to get more detailed output:

bin/logstash -f /path/to/logstash.conf --debug

bin/logstash -f /path/to/logstash.conf --config.reload.automatic

 

column 

select count(*) from myemployee

 

mysql data transform to nosql data

statement sample

SELECT    * FROM elastic.books_journal j
              LEFT JOIN elastic. books b ON b. books _id= j.product_id
              WHERE j.journal_id > :sql_last_value
                      AND j.action_time < NOW()
              ORDER BY j.journal_id 

 

jdbc_connection_string => "jdbc:mysql://localhost:3306/sampledb?useUnicode=true&characterEncoding=utf8"

 

MariaDB