Skip to content

feat: Add ls fallback support for files.File fact #1384

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: 3.x
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 145 additions & 22 deletions pyinfra/facts/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,21 @@

LINUX_STAT_COMMAND = "stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N'"
BSD_STAT_COMMAND = "stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY'"
LS_COMMAND = "ls -ld"

STAT_REGEX = (
r"user=(.*) group=(.*) mode=(.*) "
r"atime=([0-9]*) mtime=([0-9]*) ctime=([0-9]*) "
r"size=([0-9]*) (.*)"
)

# ls -ld output: permissions links user group size month day year/time path
# Supports attribute markers: . (SELinux), @ (extended attrs), + (ACL)
# Handles both "MMM DD" and "DD MMM" date formats
LS_REGEX = (
r"^([dlbcsp-][-rwxstST]{9}[.@+]?)\s+\d+\s+(\S+)\s+(\S+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.+)$"
)

FLAG_TO_TYPE = {
"b": "block",
"c": "character",
Expand Down Expand Up @@ -82,6 +90,108 @@ def _parse_datetime(value: str) -> Optional[datetime]:
return None


def _parse_ls_timestamp(month: str, day: str, year_or_time: str) -> Optional[datetime]:
"""
Parse ls timestamp format.
Examples: "Jan 1 1970", "Apr 2 2025", "Dec 31 12:34"
"""
try:
# Month abbreviation to number mapping
month_map = {
"Jan": 1,
"Feb": 2,
"Mar": 3,
"Apr": 4,
"May": 5,
"Jun": 6,
"Jul": 7,
"Aug": 8,
"Sep": 9,
"Oct": 10,
"Nov": 11,
"Dec": 12,
}

month_num = month_map.get(month)
if month_num is None:
return None

day_num = int(day)

# Check if year_or_time is a year (4 digits) or time (HH:MM)
if ":" in year_or_time:
# It's a time, assume current year
import time

current_year = time.gmtime().tm_year
hour, minute = map(int, year_or_time.split(":"))
return datetime(current_year, month_num, day_num, hour, minute)
else:
# It's a year
year_num = int(year_or_time)
return datetime(year_num, month_num, day_num)

except (ValueError, TypeError):
return None


def _parse_ls_output(output: str) -> Optional[tuple[FileDict, str]]:
"""
Parse ls -ld output and extract file information.
Example: drwxr-xr-x 1 root root 416 Jan 1 1970 /
"""
match = re.match(LS_REGEX, output.strip())
if not match:
return None

permissions = match.group(1)
user = match.group(2)
group = match.group(3)
size = match.group(4)
date_part1 = match.group(5)
date_part2 = match.group(6)
year_or_time = match.group(7)
path = match.group(8)

# Determine if it's "MMM DD" or "DD MMM" format
if date_part1.isdigit():
# "DD MMM" format (e.g., "22 Jun")
day = date_part1
month = date_part2
else:
# "MMM DD" format (e.g., "Jun 22")
month = date_part1
day = date_part2

# Extract file type from first character of permissions
path_type = FLAG_TO_TYPE[permissions[0]]

# Parse mode (skip first character which is file type, and any trailing attribute markers)
# Remove trailing attribute markers (.@+) if present
mode_str = permissions[1:10] # Take exactly 9 characters after file type
mode = _parse_mode(mode_str)

# Parse timestamp - ls shows modification time
mtime = _parse_ls_timestamp(month, day, year_or_time)

data: FileDict = {
"user": user,
"group": group,
"mode": mode,
"atime": None, # ls doesn't provide atime
"mtime": mtime,
"ctime": None, # ls doesn't provide ctime
"size": try_int(size),
}

# Handle symbolic links
if path_type == "link" and " -> " in path:
filename, target = path.split(" -> ", 1)
data["link_target"] = target.strip("'").lstrip("`")

return data, path_type


class FileDict(TypedDict):
mode: int
size: Union[int, str]
Expand Down Expand Up @@ -127,41 +237,54 @@ def command(self, path):
(
# only stat if the path exists (file or symlink)
"! (test -e {0} || test -L {0} ) || "
"( {linux_stat_command} {0} 2> /dev/null || {bsd_stat_command} {0} )"
"( {linux_stat_command} {0} 2> /dev/null || "
"{bsd_stat_command} {0} || {ls_command} {0} )"
),
path,
linux_stat_command=LINUX_STAT_COMMAND,
bsd_stat_command=BSD_STAT_COMMAND,
ls_command=LS_COMMAND,
)

@override
def process(self, output) -> Union[FileDict, Literal[False], None]:
# Try to parse as stat output first
match = re.match(STAT_REGEX, output[0])
if not match:
return None
if match:
mode = match.group(3)
path_type = FLAG_TO_TYPE[mode[0]]

mode = match.group(3)
path_type = FLAG_TO_TYPE[mode[0]]

data: FileDict = {
"user": match.group(1),
"group": match.group(2),
"mode": _parse_mode(mode[1:]),
"atime": _parse_datetime(match.group(4)),
"mtime": _parse_datetime(match.group(5)),
"ctime": _parse_datetime(match.group(6)),
"size": try_int(match.group(7)),
}
data: FileDict = {
"user": match.group(1),
"group": match.group(2),
"mode": _parse_mode(mode[1:]),
"atime": _parse_datetime(match.group(4)),
"mtime": _parse_datetime(match.group(5)),
"ctime": _parse_datetime(match.group(6)),
"size": try_int(match.group(7)),
}

if path_type != self.type:
return False

if path_type == "link":
filename = match.group(8)
filename, target = filename.split(" -> ")
data["link_target"] = target.strip("'").lstrip("`")

if path_type != self.type:
return False
return data

if path_type == "link":
filename = match.group(8)
filename, target = filename.split(" -> ")
data["link_target"] = target.strip("'").lstrip("`")
# Try to parse as ls output
ls_result = _parse_ls_output(output[0])
if ls_result is not None:
data, path_type = ls_result

return data
if path_type != self.type:
return False

return data

return None


class Link(File):
Expand Down
2 changes: 1 addition & 1 deletion tests/facts/files.Directory/file.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"arg": "/path/to/a/file",
"command": "! (test -e /path/to/a/file || test -L /path/to/a/file ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /path/to/a/file 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /path/to/a/file )",
"command": "! (test -e /path/to/a/file || test -L /path/to/a/file ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /path/to/a/file 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /path/to/a/file || ls -ld /path/to/a/file )",
"output": [
"user=pyinfra group=pyinfra mode=-rwxrwxrwx atime=1594804767 mtime=1594804767 ctime=0 size=8 '/path/to/a/file'"
],
Expand Down
2 changes: 1 addition & 1 deletion tests/facts/files.Directory/link.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"arg": "/home/pyinfra/mylink",
"command": "! (test -e /home/pyinfra/mylink || test -L /home/pyinfra/mylink ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/mylink 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/mylink )",
"command": "! (test -e /home/pyinfra/mylink || test -L /home/pyinfra/mylink ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/mylink 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/mylink || ls -ld /home/pyinfra/mylink )",
"output": [
"user=root group=root mode=lrwxrwxrwx atime=1594804774 mtime=1594804770 ctime=0 size=6 '/home/pyinfra/mylink' -> 'file.txt'"
],
Expand Down
16 changes: 16 additions & 0 deletions tests/facts/files.Directory/ls_fallback.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"arg": "/",
"command": "! (test -e / || test -L / ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' / 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' / || ls -ld / )",
"output": [
"drwxr-xr-x 1 root root 416 Jan 1 1970 /"
],
"fact": {
"group": "root",
"user": "root",
"mode": 755,
"atime": null,
"mtime": "1970-01-01T00:00:00",
"ctime": null,
"size": 416
}
}
16 changes: 16 additions & 0 deletions tests/facts/files.Directory/ls_fallback_freebsd.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"arg": "/compat/devuan01/var/hostlog",
"command": "! (test -e /compat/devuan01/var/hostlog || test -L /compat/devuan01/var/hostlog ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /compat/devuan01/var/hostlog 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /compat/devuan01/var/hostlog || ls -ld /compat/devuan01/var/hostlog )",
"output": [
"drwxr-xr-x+ 11 root wheel 420 Mar 28 16:00 /compat/devuan01/var/hostlog"
],
"fact": {
"group": "wheel",
"user": "root",
"mode": 755,
"atime": null,
"mtime": "2025-03-28T16:00:00",
"ctime": null,
"size": 420
}
}
16 changes: 16 additions & 0 deletions tests/facts/files.Directory/ls_fallback_macos.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"arg": ".",
"command": "! (test -e . || test -L . ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' . 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' . || ls -ld . )",
"output": [
"drwxr-xr-x@ 12 xonic staff 408 22 Jun 19:00 ."
],
"fact": {
"group": "staff",
"user": "xonic",
"mode": 755,
"atime": null,
"mtime": "2025-06-22T19:00:00",
"ctime": null,
"size": 408
}
}
2 changes: 1 addition & 1 deletion tests/facts/files.Directory/valid.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"arg": "/home/pyinfra/myd@-_ir",
"command": "! (test -e /home/pyinfra/myd@-_ir || test -L /home/pyinfra/myd@-_ir ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/myd@-_ir 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/myd@-_ir )",
"command": "! (test -e /home/pyinfra/myd@-_ir || test -L /home/pyinfra/myd@-_ir ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/myd@-_ir 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/myd@-_ir || ls -ld /home/pyinfra/myd@-_ir )",
"output": [
"user=pyinfra group=pyinfra mode=drw-r--r-- atime=1594804583 mtime=1594804583 ctime=0 size=0 '/home/pyinfra/myd@-_ir'"
],
Expand Down
2 changes: 1 addition & 1 deletion tests/facts/files.File/directory.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"arg": "/home/pyinfra",
"command": "! (test -e /home/pyinfra || test -L /home/pyinfra ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra )",
"command": "! (test -e /home/pyinfra || test -L /home/pyinfra ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra || ls -ld /home/pyinfra )",
"output": [
"user=root group=root mode=drw-r--r-- atime=1594804583 mtime=1594804583 ctime=0 size=0 '/home/pyinfra'"
],
Expand Down
2 changes: 1 addition & 1 deletion tests/facts/files.File/invalid_output.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"arg": "/home/pyinfra/fil-@_e.txt",
"command": "! (test -e /home/pyinfra/fil-@_e.txt || test -L /home/pyinfra/fil-@_e.txt ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/fil-@_e.txt 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/fil-@_e.txt )",
"command": "! (test -e /home/pyinfra/fil-@_e.txt || test -L /home/pyinfra/fil-@_e.txt ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/fil-@_e.txt 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/fil-@_e.txt || ls -ld /home/pyinfra/fil-@_e.txt )",
"output": [
"not-gonna-match"
],
Expand Down
2 changes: 1 addition & 1 deletion tests/facts/files.File/link.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"arg": "/home/pyinfra/mylink",
"command": "! (test -e /home/pyinfra/mylink || test -L /home/pyinfra/mylink ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/mylink 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/mylink )",
"command": "! (test -e /home/pyinfra/mylink || test -L /home/pyinfra/mylink ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/mylink 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/mylink || ls -ld /home/pyinfra/mylink )",
"output": [
"user=root group=root mode=lrwxrwxrwx atime=1594804774 mtime=1594804770 ctime=0 size=6 '/home/pyinfra/mylink' -> 'file.txt'"
],
Expand Down
16 changes: 16 additions & 0 deletions tests/facts/files.File/ls_fallback.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"arg": "/etc/hosts",
"command": "! (test -e /etc/hosts || test -L /etc/hosts ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /etc/hosts 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /etc/hosts || ls -ld /etc/hosts )",
"output": [
"-rw-r--r-- 1 root root 127 Jan 1 1970 /etc/hosts"
],
"fact": {
"group": "root",
"user": "root",
"mode": 644,
"atime": null,
"mtime": "1970-01-01T00:00:00",
"ctime": null,
"size": 127
}
}
16 changes: 16 additions & 0 deletions tests/facts/files.File/ls_fallback_selinux.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"arg": "src",
"command": "! (test -e src || test -L src ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' src 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' src || ls -ld src )",
"output": [
"-rw-r--r--. 1 src src 1024 Nov 7 19:21 src"
],
"fact": {
"group": "src",
"user": "src",
"mode": 644,
"atime": null,
"mtime": "2025-11-07T19:21:00",
"ctime": null,
"size": 1024
}
}
16 changes: 16 additions & 0 deletions tests/facts/files.File/ls_fallback_time.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"arg": "/tmp/recent_file",
"command": "! (test -e /tmp/recent_file || test -L /tmp/recent_file ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /tmp/recent_file 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /tmp/recent_file || ls -ld /tmp/recent_file )",
"output": [
"-rw-r--r-- 1 user user 1024 Jun 26 14:30 /tmp/recent_file"
],
"fact": {
"group": "user",
"user": "user",
"mode": 644,
"atime": null,
"mtime": "2025-06-26T14:30:00",
"ctime": null,
"size": 1024
}
}
2 changes: 1 addition & 1 deletion tests/facts/files.File/mode_setgid_setuid.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"arg": "file.txt",
"command": "! (test -e file.txt || test -L file.txt ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' file.txt 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' file.txt )",
"command": "! (test -e file.txt || test -L file.txt ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' file.txt 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' file.txt || ls -ld file.txt )",
"output": [
"user=pyinfra group=pyinfra mode=-rwsr-Sr-- atime=0 mtime=0 ctime=0 size=8 'file.txt'"
],
Expand Down
2 changes: 1 addition & 1 deletion tests/facts/files.File/mode_sticky.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"arg": "file.txt",
"command": "! (test -e file.txt || test -L file.txt ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' file.txt 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' file.txt )",
"command": "! (test -e file.txt || test -L file.txt ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' file.txt 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' file.txt || ls -ld file.txt )",
"output": [
"user=pyinfra group=pyinfra mode=-rw-r--r-T atime=0 mtime=0 ctime=0 size=8 'file.txt'"
],
Expand Down
2 changes: 1 addition & 1 deletion tests/facts/files.File/tilde.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"arg": "~/.bashrc",
"command": "! (test -e ~/.bashrc || test -L ~/.bashrc ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' ~/.bashrc 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' ~/.bashrc )",
"command": "! (test -e ~/.bashrc || test -L ~/.bashrc ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' ~/.bashrc 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' ~/.bashrc || ls -ld ~/.bashrc )",
"output": [
"user=pyinfra group=pyinfra mode=-rw-r--r-- atime=1723483091 mtime=1722760187 ctime=1722760187 size=4310 '/home/pyinfra/.bashrc'"
],
Expand Down
2 changes: 1 addition & 1 deletion tests/facts/files.File/tilde_with_space.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"arg": "~/My Documents/file",
"command": "! (test -e ~/'My Documents/file' || test -L ~/'My Documents/file' ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' ~/'My Documents/file' 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' ~/'My Documents/file' )",
"command": "! (test -e ~/'My Documents/file' || test -L ~/'My Documents/file' ) || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' ~/'My Documents/file' 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' ~/'My Documents/file' || ls -ld ~/'My Documents/file' )",
"output": [
"user=pyinfra group=pyinfra mode=-rw-r--r-- atime=1723484281 mtime=1723484281 ctime=1723484281 size=0 '/home/pyinfra/My Documents/file'"
],
Expand Down
Loading