diff --git a/Cargo.lock b/Cargo.lock
index b2b404c49..d3d827605 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1012,6 +1012,7 @@ dependencies = [
"futures-util",
"getrandom 0.2.7",
"git2",
+ "hostname",
"http",
"iron",
"kuchiki",
diff --git a/Cargo.toml b/Cargo.toml
index 11e7e043d..731e84efc 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -52,7 +52,8 @@ prometheus = { version = "0.13.0", default-features = false }
rustwide = "0.15.0"
mime_guess = "2"
zstd = "0.11.0"
-git2 = { version = "0.14.0", default-features = false }
+hostname = "0.3.1"
+git2 = { version = "0.14.4", default-features = false }
path-slash = "0.2.0"
once_cell = { version = "1.4.0", features = ["parking_lot"] }
base64 = "0.13"
diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs
index 4ca2a1322..0a402cfb4 100644
--- a/src/bin/cratesfyi.rs
+++ b/src/bin/cratesfyi.rs
@@ -6,7 +6,9 @@ use std::sync::Arc;
use anyhow::{anyhow, Context as _, Error, Result};
use docs_rs::db::{self, add_path_into_database, Pool, PoolClient};
use docs_rs::repositories::RepositoryStatsUpdater;
-use docs_rs::utils::{remove_crate_priority, set_crate_priority};
+use docs_rs::utils::{
+ get_config, queue_builder, remove_crate_priority, set_crate_priority, ConfigName,
+};
use docs_rs::{
BuildQueue, Config, Context, Index, Metrics, PackageKind, RustwideBuilder, Server, Storage,
};
@@ -93,6 +95,18 @@ enum CommandLine {
socket_addr: String,
},
+ StartRegistryWatcher {
+ /// Enable or disable the repository stats updater
+ #[structopt(
+ long = "repository-stats-updater",
+ default_value = "disabled",
+ possible_values(Toggle::VARIANTS)
+ )]
+ repository_stats_updater: Toggle,
+ },
+
+ StartBuildServer,
+
/// Starts the daemon
Daemon {
/// Enable or disable the registry watcher to automatically enqueue newly published crates
@@ -123,6 +137,20 @@ impl CommandLine {
match self {
Self::Build(build) => build.handle_args(ctx)?,
+ Self::StartRegistryWatcher {
+ repository_stats_updater,
+ } => {
+ if repository_stats_updater == Toggle::Enabled {
+ docs_rs::utils::daemon::start_background_repository_stats_updater(&ctx)?;
+ }
+
+ docs_rs::utils::watch_registry(ctx.build_queue()?, ctx.config()?, ctx.index()?)?;
+ }
+ Self::StartBuildServer => {
+ let build_queue = ctx.build_queue()?;
+ let rustwide_builder = RustwideBuilder::init(&ctx)?;
+ queue_builder(rustwide_builder, build_queue)?;
+ }
Self::StartWebServer { socket_addr } => {
// Blocks indefinitely
let _ = Server::start(Some(&socket_addr), &ctx)?;
@@ -336,10 +364,8 @@ impl BuildSubcommand {
.pool()?
.get()
.context("failed to get a database connection")?;
- let res =
- conn.query("SELECT * FROM config WHERE name = 'rustc_version';", &[])?;
- if !res.is_empty() {
+ if get_config::<String>(&mut conn, ConfigName::RustcVersion)?.is_some() {
println!("update-toolchain was already called in the past, exiting");
return Ok(());
}
diff --git a/src/build_queue.rs b/src/build_queue.rs
index 2aac34e46..5fc031e1e 100644
--- a/src/build_queue.rs
+++ b/src/build_queue.rs
@@ -2,15 +2,14 @@ use crate::db::{delete_crate, Pool};
use crate::docbuilder::PackageKind;
use crate::error::Result;
use crate::storage::Storage;
-use crate::utils::{get_crate_priority, report_error};
+use crate::utils::{get_config, get_crate_priority, report_error, set_config, ConfigName};
use crate::{Config, Index, Metrics, RustwideBuilder};
use anyhow::Context;
use crates_index_diff::Change;
use log::{debug, info};
-use std::fs;
-use std::path::PathBuf;
+use git2::Oid;
use std::sync::Arc;
#[derive(Debug, Clone, Eq, PartialEq, serde::Serialize)]
@@ -48,6 +47,24 @@ impl BuildQueue {
}
}
+ pub fn last_seen_reference(&self) -> Result<Option<Oid>> {
+ let mut conn = self.db.get()?;
+ if let Some(value) = get_config::<String>(&mut conn, ConfigName::LastSeenIndexReference)? {
+ return Ok(Some(Oid::from_str(&value)?));
+ }
+ Ok(None)
+ }
+
+ fn set_last_seen_reference(&self, oid: Oid) -> Result<()> {
+ let mut conn = self.db.get()?;
+ set_config(
+ &mut conn,
+ ConfigName::LastSeenIndexReference,
+ oid.to_string(),
+ )?;
+ Ok(())
+ }
+
pub fn add_crate(
&self,
name: &str,
@@ -118,14 +135,36 @@ impl BuildQueue {
f: impl FnOnce(&QueuedCrate) -> Result<()>,
) -> Result<()> {
let mut conn = self.db.get()?;
-
- let queued = self.queued_crates()?;
- let to_process = match queued.get(0) {
+ let mut transaction = conn.transaction()?;
+
+ // fetch the next available crate from the queue table.
+ // We are using `SELECT FOR UPDATE` inside a transaction so
+ // the QueuedCrate is locked until we are finished with it.
+ // `SKIP LOCKED` here will enable another build-server to just
+ // skip over taken (=locked) rows and start building the first
+ // available one.
+ let to_process = match transaction
+ .query_opt(
+ "SELECT id, name, version, priority, registry
+ FROM queue
+ WHERE attempt < $1
+ ORDER BY priority ASC, attempt ASC, id ASC
+ LIMIT 1
+ FOR UPDATE SKIP LOCKED",
+ &[&self.max_attempts],
+ )?
+ .map(|row| QueuedCrate {
+ id: row.get("id"),
+ name: row.get("name"),
+ version: row.get("version"),
+ priority: row.get("priority"),
+ registry: row.get("registry"),
+ }) {
Some(krate) => krate,
None => return Ok(()),
};
- let res = f(to_process).with_context(|| {
+ let res = f(&to_process).with_context(|| {
format!(
"Failed to build package {}-{} from queue",
to_process.name, to_process.version
@@ -134,15 +173,16 @@ impl BuildQueue {
self.metrics.total_builds.inc();
match res {
Ok(()) => {
- conn.execute("DELETE FROM queue WHERE id = $1;", &[&to_process.id])?;
+ transaction.execute("DELETE FROM queue WHERE id = $1;", &[&to_process.id])?;
}
Err(e) => {
// Increase attempt count
- let rows = conn.query(
- "UPDATE queue SET attempt = attempt + 1 WHERE id = $1 RETURNING attempt;",
- &[&to_process.id],
- )?;
- let attempt: i32 = rows[0].get(0);
+ let attempt: i32 = transaction
+ .query_one(
+ "UPDATE queue SET attempt = attempt + 1 WHERE id = $1 RETURNING attempt;",
+ &[&to_process.id],
+ )?
+ .get(0);
if attempt >= self.max_attempts {
self.metrics.failed_builds.inc();
@@ -152,39 +192,31 @@ impl BuildQueue {
}
}
+ transaction.commit()?;
+
Ok(())
}
}
/// Locking functions.
impl BuildQueue {
- pub(crate) fn lock_path(&self) -> PathBuf {
- self.config.prefix.join("docsrs.lock")
- }
+ /// Checks for the lock and returns whether it currently exists.
+ pub fn is_locked(&self) -> Result<bool> {
+ let mut conn = self.db.get()?;
- /// Checks for the lock file and returns whether it currently exists.
- pub fn is_locked(&self) -> bool {
- self.lock_path().exists()
+ Ok(get_config::<bool>(&mut conn, ConfigName::QueueLocked)?.unwrap_or(false))
}
- /// Creates a lock file. Daemon will check this lock file and stop operating if it exists.
+ /// lock the queue. Daemon will check this lock and stop operating if it exists.
pub fn lock(&self) -> Result<()> {
- let path = self.lock_path();
- if !path.exists() {
- fs::OpenOptions::new().write(true).create(true).open(path)?;
- }
-
- Ok(())
+ let mut conn = self.db.get()?;
+ set_config(&mut conn, ConfigName::QueueLocked, true)
}
- /// Removes lock file.
+ /// unlock the queue.
pub fn unlock(&self) -> Result<()> {
- let path = self.lock_path();
- if path.exists() {
- fs::remove_file(path)?;
- }
-
- Ok(())
+ let mut conn = self.db.get()?;
+ set_config(&mut conn, ConfigName::QueueLocked, false)
}
}
@@ -266,6 +298,13 @@ impl BuildQueue {
}
}
+ // additionally set the reference in the database
+ // so this survives recreating the registry watcher
+ // server.
+ self.set_last_seen_reference(oid)?;
+
+ // store the last seen reference as git reference in
+ // the local crates.io index repo.
diff.set_last_seen_reference(oid)?;
Ok(crates_added)
@@ -559,4 +598,54 @@ mod tests {
Ok(())
});
}
+
+ #[test]
+ fn test_last_seen_reference_in_db() {
+ crate::test::wrapper(|env| {
+ let queue = env.build_queue();
+ queue.unlock()?;
+ assert!(!queue.is_locked()?);
+ // initial db ref is empty
+ assert_eq!(queue.last_seen_reference()?, None);
+ assert!(!queue.is_locked()?);
+
+ let oid = git2::Oid::from_str("ffffffff")?;
+ queue.set_last_seen_reference(oid)?;
+
+ assert_eq!(queue.last_seen_reference()?, Some(oid));
+ assert!(!queue.is_locked()?);
+
+ Ok(())
+ });
+ }
+
+ #[test]
+ fn test_broken_db_reference_breaks() {
+ crate::test::wrapper(|env| {
+ let mut conn = env.db().conn();
+ set_config(&mut conn, ConfigName::LastSeenIndexReference, "invalid")?;
+
+ let queue = env.build_queue();
+ assert!(queue.last_seen_reference().is_err());
+
+ Ok(())
+ });
+ }
+
+ #[test]
+ fn test_queue_lock() {
+ crate::test::wrapper(|env| {
+ let queue = env.build_queue();
+ // unlocked without config
+ assert!(!queue.is_locked()?);
+
+ queue.lock()?;
+ assert!(queue.is_locked()?);
+
+ queue.unlock()?;
+ assert!(!queue.is_locked()?);
+
+ Ok(())
+ });
+ }
}
diff --git a/src/db/add_package.rs b/src/db/add_package.rs
index 60e9e124f..b0cbaa5ed 100644
--- a/src/db/add_package.rs
+++ b/src/db/add_package.rs
@@ -186,14 +186,15 @@ pub(crate) fn add_build_into_database(
) -> Result<i32> {
debug!("Adding build into database");
let rows = conn.query(
- "INSERT INTO builds (rid, rustc_version, docsrs_version, build_status)
- VALUES ($1, $2, $3, $4)
+ "INSERT INTO builds (rid, rustc_version, docsrs_version, build_status, build_server)
+ VALUES ($1, $2, $3, $4, $5)
RETURNING id",
&[
&release_id,
&res.rustc_version,
&res.docsrs_version,
&res.successful,
+ &hostname::get()?.to_str().unwrap_or(""),
],
)?;
Ok(rows[0].get(0))
diff --git a/src/db/migrate.rs b/src/db/migrate.rs
index d175059ab..fb203605e 100644
--- a/src/db/migrate.rs
+++ b/src/db/migrate.rs
@@ -837,7 +837,13 @@ pub fn migrate(version: Option<Version>, conn: &mut Client) -> crate::error::Res
)
.map(|_| ())
}
- )
+ ),
+ sql_migration!(
+ context, 33, "add hostname to build-table",
+ "ALTER TABLE builds ADD COLUMN build_server TEXT NOT NULL DEFAULT '';",
+ "ALTER TABLE builds DROP COLUMN build_server;",
+ ),
+
];
for migration in migrations {
diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs
index e591f6fa0..97d653909 100644
--- a/src/docbuilder/rustwide_builder.rs
+++ b/src/docbuilder/rustwide_builder.rs
@@ -8,7 +8,9 @@ use crate::error::Result;
use crate::index::api::ReleaseData;
use crate::repositories::RepositoryStatsUpdater;
use crate::storage::{rustdoc_archive_path, source_archive_path};
-use crate::utils::{copy_dir_all, parse_rustc_version, queue_builder, CargoMetadata};
+use crate::utils::{
+ copy_dir_all, parse_rustc_version, queue_builder, set_config, CargoMetadata, ConfigName,
+};
use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage};
use crate::{Config, Context, Index, Metrics, Storage};
use anyhow::{anyhow, bail, Error};
@@ -20,7 +22,6 @@ use rustwide::cmd::{Command, CommandError, SandboxBuilder, SandboxImage};
use rustwide::logging::{self, LogStorage};
use rustwide::toolchain::ToolchainError;
use rustwide::{AlternativeRegistry, Build, Crate, Toolchain, Workspace, WorkspaceBuilder};
-use serde_json::Value;
use std::collections::{HashMap, HashSet};
use std::path::Path;
use std::sync::Arc;
@@ -225,12 +226,12 @@ impl RustwideBuilder {
.tempdir()?;
copy_dir_all(source, &dest)?;
add_path_into_database(&self.storage, "", &dest)?;
- conn.query(
- "INSERT INTO config (name, value) VALUES ('rustc_version', $1) \
- ON CONFLICT (name) DO UPDATE SET value = $1;",
- &[&Value::String(self.rustc_version.clone())],
- )?;
+ set_config(
+ &mut conn,
+ ConfigName::RustcVersion,
+ self.rustc_version.clone(),
+ )?;
Ok(())
})()
.map_err(|e| failure::Error::from_boxed_compat(e.into()))
@@ -806,6 +807,7 @@ pub(crate) struct BuildResult {
mod tests {
use super::*;
use crate::test::{assert_redirect, assert_success, wrapper};
+ use serde_json::Value;
#[test]
#[ignore]
diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs
index 7dc0c11e1..292cd50c5 100644
--- a/src/metrics/mod.rs
+++ b/src/metrics/mod.rs
@@ -158,7 +158,7 @@ impl Metrics {
self.idle_db_connections.set(pool.idle_connections() as i64);
self.used_db_connections.set(pool.used_connections() as i64);
self.max_db_connections.set(pool.max_size() as i64);
- self.queue_is_locked.set(queue.is_locked() as i64);
+ self.queue_is_locked.set(queue.is_locked()? as i64);
self.queued_crates_count.set(queue.pending_count()? as i64);
self.prioritized_crates_count
diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs
index f98413004..66389f306 100644
--- a/src/utils/daemon.rs
+++ b/src/utils/daemon.rs
@@ -4,13 +4,62 @@
use crate::{
utils::{queue_builder, report_error},
- Context, RustwideBuilder,
+ BuildQueue, Config, Context, Index, RustwideBuilder,
};
use anyhow::{anyhow, Context as _, Error};
use log::{debug, info};
+use std::sync::Arc;
use std::thread;
use std::time::{Duration, Instant};
+/// Run the registry watcher
+/// NOTE: this should only be run once, otherwise crates would be added
+/// to the queue multiple times.
+pub fn watch_registry(
+ build_queue: Arc<BuildQueue>,
+ config: Arc<Config>,
+ index: Arc<Index>,
+) -> Result<(), Error> {
+ let mut last_gc = Instant::now();
+
+ // On startup we fetch the last seen index reference from
+ // the database and set it in the local index repository.
+ match build_queue.last_seen_reference() {
+ Ok(Some(oid)) => {
+ index.diff()?.set_last_seen_reference(oid)?;
+ }
+ Ok(None) => {}
+ Err(err) => {
+ log::error!(
+ "queue locked because of invalid last_seen_index_reference in database: {:?}",
+ err
+ );
+ build_queue.lock()?;
+ }
+ }
+
+ loop {
+ if build_queue.is_locked()? {
+ debug!("Queue is locked, skipping checking new crates");
+ } else {
+ debug!("Checking new crates");
+ match build_queue
+ .get_new_crates(&index)
+ .context("Failed to get new crates")
+ {
+ Ok(n) => debug!("{} crates added to queue", n),
+ Err(e) => report_error(&e),
+ }
+ }
+
+ if last_gc.elapsed().as_secs() >= config.registry_gc_interval {
+ index.run_git_gc();
+ last_gc = Instant::now();
+ }
+ thread::sleep(Duration::from_secs(60));
+ }
+}
+
fn start_registry_watcher(context: &dyn Context) -> Result<(), Error> {
let build_queue = context.build_queue()?;
let config = context.config()?;
@@ -22,32 +71,29 @@ fn start_registry_watcher(context: &dyn Context) -> Result<(), Error> {
// space this out to prevent it from clashing against the queue-builder thread on launch
thread::sleep(Duration::from_secs(30));
- let mut last_gc = Instant::now();
- loop {
- if build_queue.is_locked() {
- debug!("Lock file exists, skipping checking new crates");
- } else {
- debug!("Checking new crates");
- match build_queue
- .get_new_crates(&index)
- .context("Failed to get new crates")
- {
- Ok(n) => debug!("{} crates added to queue", n),
- Err(e) => report_error(&e),
- }
- }
-
- if last_gc.elapsed().as_secs() >= config.registry_gc_interval {
- index.run_git_gc();
- last_gc = Instant::now();
- }
- thread::sleep(Duration::from_secs(60));
- }
+ watch_registry(build_queue, config, index)
})?;
Ok(())
}
+pub fn start_background_repository_stats_updater(context: &dyn Context) -> Result<(), Error> {
+ // This call will still skip github repositories updates and continue if no token is provided
+ // (gitlab doesn't require to have a token). The only time this can return an error is when
+ // creating a pool or if config fails, which shouldn't happen here because this is run right at
+ // startup.
+ let updater = context.repository_stats_updater()?;
+ cron(
+ "repositories stats updater",
+ Duration::from_secs(60 * 60),
+ move || {
+ updater.update_all_crates()?;
+ Ok(())
+ },
+ )?;
+ Ok(())
+}
+
pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Result<(), Error> {
// Start the web server before doing anything more expensive
// Please check with an administrator before changing this (see #1172 for context).
@@ -70,19 +116,7 @@ pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Res
})
.unwrap();
- // This call will still skip github repositories updates and continue if no token is provided
- // (gitlab doesn't require to have a token). The only time this can return an error is when
- // creating a pool or if config fails, which shouldn't happen here because this is run right at
- // startup.
- let updater = context.repository_stats_updater()?;
- cron(
- "repositories stats updater",
- Duration::from_secs(60 * 60),
- move || {
- updater.update_all_crates()?;
- Ok(())
- },
- )?;
+ start_background_repository_stats_updater(context)?;
// Never returns; `server` blocks indefinitely when dropped
// NOTE: if a anyhow occurred earlier in `start_daemon`, the server will _not_ be joined -
diff --git a/src/utils/mod.rs b/src/utils/mod.rs
index 9522ed720..a95894b94 100644
--- a/src/utils/mod.rs
+++ b/src/utils/mod.rs
@@ -2,7 +2,7 @@
pub(crate) use self::cargo_metadata::{CargoMetadata, Package as MetadataPackage};
pub(crate) use self::copy::copy_dir_all;
-pub use self::daemon::start_daemon;
+pub use self::daemon::{start_daemon, watch_registry};
pub(crate) use self::html::rewrite_lol;
pub use self::queue::{get_crate_priority, remove_crate_priority, set_crate_priority};
pub use self::queue_builder::queue_builder;
@@ -15,11 +15,15 @@ mod cargo_metadata;
#[cfg(feature = "consistency_check")]
pub mod consistency;
mod copy;
-pub(crate) mod daemon;
+pub mod daemon;
mod html;
mod queue;
pub(crate) mod queue_builder;
mod rustc_version;
+use anyhow::Result;
+use postgres::Client;
+use serde::de::DeserializeOwned;
+use serde::Serialize;
pub(crate) mod sized_buffer;
pub(crate) const APP_USER_AGENT: &str = concat!(
@@ -36,3 +40,87 @@ pub(crate) fn report_error(err: &anyhow::Error) {
log::error!("{:?}", err);
}
}
+
+#[derive(strum::IntoStaticStr)]
+#[strum(serialize_all = "snake_case")]
+pub enum ConfigName {
+ RustcVersion,
+ LastSeenIndexReference,
+ QueueLocked,
+}
+
+pub fn set_config(
+ conn: &mut Client,
+ name: ConfigName,
+ value: impl Serialize,
+) -> anyhow::Result<()> {
+ let name: &'static str = name.into();
+ conn.execute(
+ "INSERT INTO config (name, value)
+ VALUES ($1, $2)
+ ON CONFLICT (name) DO UPDATE SET value = $2;",
+ &[&name, &serde_json::to_value(value)?],
+ )?;
+ Ok(())
+}
+
+pub fn get_config<T>(conn: &mut Client, name: ConfigName) -> Result<Option<T>>
+where
+ T: DeserializeOwned,
+{
+ let name: &'static str = name.into();
+ Ok(
+ match conn.query_opt("SELECT value FROM config WHERE name = $1;", &[&name])? {
+ Some(row) => serde_json::from_value(row.get("value"))?,
+ None => None,
+ },
+ )
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::test::wrapper;
+ use serde_json::Value;
+ use test_case::test_case;
+
+ #[test_case(ConfigName::RustcVersion, "rustc_version")]
+ #[test_case(ConfigName::QueueLocked, "queue_locked")]
+ #[test_case(ConfigName::LastSeenIndexReference, "last_seen_index_reference")]
+ fn test_configname_variants(variant: ConfigName, expected: &'static str) {
+ let name: &'static str = variant.into();
+ assert_eq!(name, expected);
+ }
+
+ #[test]
+ fn test_get_config_empty() {
+ wrapper(|env| {
+ let mut conn = env.db().conn();
+ conn.execute("DELETE FROM config", &[])?;
+
+ assert!(get_config::<String>(&mut conn, ConfigName::RustcVersion)?.is_none());
+ Ok(())
+ });
+ }
+
+ #[test]
+ fn test_set_and_get_config_() {
+ wrapper(|env| {
+ let mut conn = env.db().conn();
+ conn.execute("DELETE FROM config", &[])?;
+
+ assert!(get_config::<String>(&mut conn, ConfigName::RustcVersion)?.is_none());
+
+ set_config(
+ &mut conn,
+ ConfigName::RustcVersion,
+ Value::String("some value".into()),
+ )?;
+ assert_eq!(
+ get_config(&mut conn, ConfigName::RustcVersion)?,
+ Some("some value".to_string())
+ );
+ Ok(())
+ });
+ }
+}
diff --git a/src/utils/queue_builder.rs b/src/utils/queue_builder.rs
index bfb91ad36..bbcbf56b1 100644
--- a/src/utils/queue_builder.rs
+++ b/src/utils/queue_builder.rs
@@ -1,6 +1,6 @@
use crate::{docbuilder::RustwideBuilder, utils::report_error, BuildQueue};
-use anyhow::Error;
-use log::{debug, error, info, warn};
+use anyhow::{Context, Error};
+use log::{debug, error, warn};
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::sync::Arc;
use std::time::Duration;
@@ -8,71 +8,49 @@ use std::{fs, io, thread};
pub(crate) const TEMPDIR_PREFIX: &str = "docsrs-docs";
-// TODO: change to `fn() -> Result<!, Error>` when never _finally_ stabilizes
pub fn queue_builder(
mut builder: RustwideBuilder,
build_queue: Arc<BuildQueue>,
) -> Result<(), Error> {
- /// Represents the current state of the builder thread.
- enum BuilderState {
- /// The builder thread has just started, and hasn't built any crates yet.
- Fresh,
- /// The builder has just seen an empty build queue.
- EmptyQueue,
- /// The builder has just seen the lock file.
- Locked,
- /// The builder has started (or just finished) building a crate.
- QueueInProgress,
- }
-
- let mut status = BuilderState::Fresh;
-
loop {
if let Err(e) = remove_tempdirs() {
report_error(&anyhow::anyhow!(e).context("failed to remove temporary directories"));
}
- if !matches!(status, BuilderState::QueueInProgress) {
- thread::sleep(Duration::from_secs(60));
- }
-
// check lock file
- if build_queue.is_locked() {
- warn!("Lock file exists, skipping building new crates");
- status = BuilderState::Locked;
- continue;
- }
-
- // Only build crates if there are any to build
- debug!("Checking build queue");
- match build_queue.pending_count() {
- Err(e) => {
- report_error(&e.context("Failed to read the number of crates in the queue"));
+ match build_queue.is_locked().context("could not get queue lock") {
+ Ok(true) => {
+ warn!("Build queue is locked, skipping building new crates");
+ thread::sleep(Duration::from_secs(60));
continue;
}
-
- Ok(0) => {
- debug!("Queue is empty, going back to sleep");
- status = BuilderState::EmptyQueue;
+ Ok(false) => {}
+ Err(err) => {
+ report_error(&err);
+ thread::sleep(Duration::from_secs(60));
continue;
}
-
- Ok(queue_count) => info!("Starting build with {} crates in queue", queue_count),
}
- status = BuilderState::QueueInProgress;
-
// If a panic occurs while building a crate, lock the queue until an admin has a chance to look at it.
+ debug!("Checking build queue");
let res = catch_unwind(AssertUnwindSafe(|| {
- if let Err(e) = build_queue.build_next_queue_package(&mut builder) {
- report_error(&e.context("Failed to build crate from queue"));
+ match build_queue.build_next_queue_package(&mut builder) {
+ Ok(true) => {}
+ Ok(false) => {
+ debug!("Queue is empty, going back to sleep");
+ thread::sleep(Duration::from_secs(60));
+ }
+ Err(e) => {
+ report_error(&e.context("Failed to build crate from queue"));
+ }
}
}));
if let Err(e) = res {
error!("GRAVE ERROR Building new crates panicked: {:?}", e);
- // If we panic here something is really truly wrong and trying to handle the error won't help.
- build_queue.lock().expect("failed to lock queue");
+ thread::sleep(Duration::from_secs(60));
+ continue;
}
}
}
diff --git a/src/web/page/templates.rs b/src/web/page/templates.rs
index 7c89f6fff..fb98abe1e 100644
--- a/src/web/page/templates.rs
+++ b/src/web/page/templates.rs
@@ -1,4 +1,7 @@
-use crate::error::Result;
+use crate::{
+ error::Result,
+ utils::{get_config, ConfigName},
+};
use anyhow::Context;
use chrono::{DateTime, Utc};
use path_slash::PathExt;
@@ -31,19 +34,8 @@ impl TemplateData {
}
fn load_rustc_resource_suffix(conn: &mut Client) -> Result<String> {
- let res = conn.query(
- "SELECT value FROM config WHERE name = 'rustc_version';",
- &[],
- )?;
-
- if res.is_empty() {
- anyhow::bail!("missing rustc version");
- }
-
- if let Ok(vers) = res[0].try_get::<_, Value>("value") {
- if let Some(vers_str) = vers.as_str() {
- return crate::utils::parse_rustc_version(vers_str);
- }
+ if let Some(vers_str) = get_config::<String>(conn, ConfigName::RustcVersion)? {
+ return crate::utils::parse_rustc_version(vers_str);
}
anyhow::bail!("failed to parse the rustc version");
diff --git a/src/web/sitemap.rs b/src/web/sitemap.rs
index ef4f69d3e..92da7239f 100644
--- a/src/web/sitemap.rs
+++ b/src/web/sitemap.rs
@@ -1,4 +1,11 @@
-use crate::{db::Pool, docbuilder::Limits, impl_webpage, web::error::Nope, web::page::WebPage};
+use crate::{
+ db::Pool,
+ docbuilder::Limits,
+ impl_webpage,
+ utils::{get_config, ConfigName},
+ web::error::Nope,
+ web::page::WebPage,
+};
use chrono::{DateTime, Utc};
use iron::{
headers::ContentType,
@@ -7,7 +14,6 @@ use iron::{
};
use router::Router;
use serde::Serialize;
-use serde_json::Value;
/// sitemap index
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
@@ -102,19 +108,12 @@ impl_webpage!(AboutBuilds = "core/about/builds.html");
pub fn about_builds_handler(req: &mut Request) -> IronResult<Response> {
let mut conn = extension!(req, Pool).get()?;
- let res = ctry!(
+
+ let rustc_version = ctry!(
req,
- conn.query("SELECT value FROM config WHERE name = 'rustc_version'", &[]),
+ get_config::<String>(&mut conn, ConfigName::RustcVersion)
);
- let rustc_version = res.get(0).and_then(|row| {
- if let Ok(Some(Value::String(version))) = row.try_get(0) {
- Some(version)
- } else {
- None
- }
- });
-
AboutBuilds {
rustc_version,
limits: Limits::default(),