fix: page/graph load speed: part 1 (#6611)

* fix: page/graph load speed: part 1
Speed up load_all_os() by caching the full $config['os'] array
Function time goes from 1.13648s -> 0.00073s
The cache is updated during discovery, but only if it is >22hrs old.
Additionally, the cache is not used and a full yaml load is forced if the cache is >24hrs old.
The cache time is controlled by $config['os_def_cache_time'], I don't see any reason anyone would change that time.

* Delete Session.php

* Include the cache directory.
This commit is contained in:
Tony Murray 2017-05-10 02:57:10 -05:00 committed by Neil Lathwood
parent d60c43ed8d
commit b6e18db766
6 changed files with 66 additions and 25 deletions

4
cache/.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
# Ignore everything in this directory
*
# Except this file
!.gitignore

View File

@ -118,6 +118,8 @@ if (get_lock('schema') === false) {
require 'includes/sql-schema/update.php';
}
update_os_cache(); // will only update if needed
$discovered_devices = 0;
if (!empty($config['distributed_poller_group'])) {

View File

@ -1576,30 +1576,63 @@ function load_os(&$device)
}
/**
* @param array $restricted
* Load all OS, optionally load just the OS used by existing devices
* Default cache time is 1 day. Controlled by os_def_cache_time.
*
* @param bool $existing Only load OS that have existing OS in the database
* @param bool $cached Load os definitions from the cache file
*/
function load_all_os($restricted = array())
function load_all_os($existing = false, $cached = true)
{
global $config;
if (!empty($restricted)) {
$list = $restricted;
$cache_file = $config['install_dir'] . '/cache/os_defs.cache';
if ($cached && is_file($cache_file) && (time() - filemtime($cache_file) < $config['os_def_cache_time'])) {
// Cached
$os_defs = unserialize(file_get_contents($cache_file));
if ($existing) {
// remove unneeded os
$os_defs = array_diff_key($os_defs, dbFetchColumn('SELECT DISTINCT(`os`) FROM `devices`'));
}
$config['os'] = array_replace_recursive($os_defs, $config['os']);
} else {
$list = glob($config['install_dir'].'/includes/definitions/*.yaml');
if (count($list) == count($config['os'])) {
// already fully loaded
return;
// load from yaml
if ($existing) {
$os_list = array_map(function ($os) use ($config) {
return $config['install_dir'] . '/includes/definitions/'. $os . '.yaml';
}, dbFetchColumn('SELECT DISTINCT(`os`) FROM `devices`'));
} else {
$os_list = glob($config['install_dir'].'/includes/definitions/*.yaml');
}
foreach ($os_list as $file) {
$tmp = Symfony\Component\Yaml\Yaml::parse(file_get_contents($file));
if (isset($config['os'][$tmp['os']])) {
$config['os'][$tmp['os']] = array_replace_recursive($tmp, $config['os'][$tmp['os']]);
} else {
$config['os'][$tmp['os']] = $tmp;
}
}
}
}
foreach ($list as $file) {
$tmp = Symfony\Component\Yaml\Yaml::parse(
file_get_contents($file)
);
if (isset($config['os'][$tmp['os']])) {
$config['os'][$tmp['os']] = array_replace_recursive($tmp, $config['os'][$tmp['os']]);
} else {
$config['os'][$tmp['os']] = $tmp;
}
/**
* Update the OS cache file cache/os_defs.cache
*/
function update_os_cache()
{
global $config;
$cache_file = $config['install_dir'] . '/cache/os_defs.cache';
$cache_keep_time = $config['os_def_cache_time'] - 7200; // 2hr buffer
if (!is_file($cache_file) || time() - filemtime($cache_file) > $cache_keep_time) {
d_echo('Updating os_def.cache... ');
load_all_os(false, false);
file_put_contents($cache_file, serialize($config['os']));
d_echo("Done\n");
}
}
@ -1704,7 +1737,7 @@ function get_user_pref($name, $default = null, $user_id = null)
{
global $user_prefs;
if (array_key_exists($name, $user_prefs)) {
if (is_array($user_prefs) && array_key_exists($name, $user_prefs)) {
return $user_prefs[$name];
}

View File

@ -191,6 +191,9 @@ $config['snmp']['v3'][0]['cryptoalgo'] = 'AES';
// Devices must respond to icmp by default
$config['icmp_check'] = true;
// The amount of time to keep the OS cache
$config['os_def_cache_time'] = 86400;
// Autodiscovery Settings
$config['autodiscovery']['xdp'] = true;
$config['autodiscovery']['xdp_exclude']['sysdesc_regexp'][] = '/-K9W8-/'; // Cisco Lightweight Access Point

View File

@ -152,12 +152,8 @@ if (module_selected('web', $init_modules)) {
$config['title_image'] = 'images/librenms_logo_'.$config['site_style'].'.svg';
}
require $install_dir . '/html/includes/vars.inc.php';
$tmp_list = dbFetchRows('SELECT DISTINCT(`os`) FROM `devices`');
$os_list = array();
foreach ($tmp_list as $k => $v) {
$os_list[] = $config['install_dir'].'/includes/definitions/'. $v['os'] . '.yaml';
}
load_all_os($os_list);
load_all_os(true);
}
$console_color = new Console_Color2();

View File

@ -143,7 +143,10 @@ class OSDiscoveryTest extends \PHPUnit_Framework_TestCase
{
global $config;
load_all_os(); // make sure all OS are loaded
// make sure all OS are loaded
if (count($config['os']) < count(glob($config['install_dir'].'/includes/definitions/*.yaml'))) {
load_all_os();
}
$excluded_os = array(
'default',